Start Converting URLs to Markdown in Under 60 Seconds

import requests

def fetch_for_rag(url: str) -> dict:
    """Fetch clean markdown optimized for RAG pipelines"""
    response = requests.get(
        f"https://api.neoreader.dev/{url}",
        headers={
            "X-API-Key": "YOUR_API_KEY",
            "X-Respond-With": "markdown",
            "X-Target-Selector": "article, main",
            "X-Remove-Selector": "nav, footer, .ads"
        }
    )

    if response.status_code == 200:
        return {
            "content": response.text,
            "source": url,
            "tokens": len(response.text.split())
        }

    raise Exception(f"Error: {response.status_code}")

# Feed to vector database
content = fetch_for_rag("https://docs.example.com/guide")
vector_db.upsert(
    id=content["source"],
    text=content["content"],
    metadata={"source": content["source"]}
)

async function fetchDocsForRAG(urls) {
  const results = await Promise.all(
    urls.map(async (url) => {
      const response = await fetch(
        `https://api.neoreader.dev/${url}`,
        {
          headers: {
            'X-API-Key': 'YOUR_API_KEY',
            'X-Respond-With': 'markdown',
            'X-Target-Selector': 'article, main'
          }
        }
      );

      const content = await response.text();
      return { url, content, tokens: content.split(' ').length };
    })
  );

  return results;
}

// Process 100 docs concurrently
const docs = await fetchDocsForRAG(docUrls);
await vectorDB.upsert(docs);

import asyncio
import aiohttp

async def scrape_news_articles(urls: list[str]) -> list[dict]:
    """Scrape multiple news articles concurrently"""
    async with aiohttp.ClientSession() as session:
        tasks = []

        for url in urls:
            headers = {
                "X-API-Key": "YOUR_API_KEY",
                "X-Respond-With": "markdown",
                "X-Target-Selector": "article, [itemtype*='Article']",
                "X-Remove-Selector": ".ad, .social-share, .comments",
                "X-With-Images-Summary": "true"
            }

            task = session.get(
                f"https://api.neoreader.dev/{url}",
                headers=headers
            )
            tasks.append(task)

        responses = await asyncio.gather(*tasks, return_exceptions=True)

        articles = []
        for url, response in zip(urls, responses):
            if isinstance(response, Exception):
                continue

            content = await response.text()
            articles.append({
                "url": url,
                "content": content,
                "timestamp": datetime.now().isoformat()
            })

        return articles

# Scrape 10,000 articles daily
articles = await scrape_news_articles(news_urls)

import requests

def scrape_spa(url: str) -> str:
    """
    Automatically detects SPAs and renders them.
    Supports: React, Vue, Next.js, Nuxt, Angular, Svelte
    """
    response = requests.get(
        f"https://api.neoreader.dev/{url}",
        headers={
            "X-API-Key": "YOUR_API_KEY",
            "X-Respond-With": "markdown",
            # No X-Wait-For-Selector needed!
            # Automatic framework detection handles it
        }
    )

    return response.text

# Works automatically for any framework
react_content = scrape_spa("https://react-app.com")
vue_content = scrape_spa("https://vue-app.com")
next_content = scrape_spa("https://nextjs-app.com")

async function scrapeDynamicContent(url, options = {}) {
  const response = await fetch(
    `https://api.neoreader.dev/${url}`,
    {
      headers: {
        'X-API-Key': 'YOUR_API_KEY',
        'X-Respond-With': 'markdown',
        // Optional: explicit wait for custom elements
        'X-Wait-For-Selector': options.waitFor || '',
        'X-Timeout': options.timeout || '30'
      }
    }
  );

  return await response.text();
}

// Automatic detection
const content1 = await scrapeDynamicContent('https://react-app.com');

// Custom wait strategy
const content2 = await scrapeDynamicContent('https://custom-spa.com', {
  waitFor: '[data-loaded="true"]',
  timeout: 60
});

from langchain.document_loaders.base import BaseLoader
from langchain.schema import Document
import requests
from typing import List

class NeoReaderLoader(BaseLoader):
    """LangChain document loader using Neo Reader API"""

    def __init__(
        self,
        urls: List[str],
        api_key: str,
        target_selector: str = "article, main"
    ):
        self.urls = urls
        self.api_key = api_key
        self.target_selector = target_selector

    def load(self) -> List[Document]:
        """Load documents from URLs"""
        documents = []

        for url in self.urls:
            response = requests.get(
                f"https://api.neoreader.dev/{url}",
                headers={
                    "X-API-Key": self.api_key,
                    "X-Respond-With": "markdown",
                    "X-Target-Selector": self.target_selector,
                    "X-Remove-Selector": "nav, footer, .ads"
                }
            )

            if response.status_code == 200:
                documents.append(Document(
                    page_content=response.text,
                    metadata={
                        "source": url,
                        "loader": "neo-reader"
                    }
                ))

        return documents

# Usage with LangChain
loader = NeoReaderLoader(
    urls=["https://docs.example.com/guide"],
    api_key="YOUR_API_KEY"
)

documents = loader.load()
vector_store = FAISS.from_documents(documents, embeddings)

import { BaseLoader } from 'langchain/document_loaders/base';
import { Document } from 'langchain/document';

interface NeoReaderOptions {
  apiKey: string;
  targetSelector?: string;
  removeSelector?: string;
}

class NeoReaderLoader extends BaseLoader {
  private urls: string[];
  private options: NeoReaderOptions;

  constructor(urls: string[], options: NeoReaderOptions) {
    super();
    this.urls = urls;
    this.options = options;
  }

  async load(): Promise<Document[]> {
    const documents = await Promise.all(
      this.urls.map(async (url) => {
        const response = await fetch(
          `https://api.neoreader.dev/${url}`,
          {
            headers: {
              'Authorization': `Bearer ${this.options.apiKey}`,
              'X-Respond-With': 'markdown',
              'X-Target-Selector': this.options.targetSelector || 'article',
              'X-Remove-Selector': this.options.removeSelector || ''
            }
          }
        );

        const content = await response.text();

        return new Document({
          pageContent: content,
          metadata: { source: url }
        });
      })
    );

    return documents;
  }
}

import requests

def capture_screenshot(url: str, full_page: bool = False) -> str:
    """
    Capture screenshot of any webpage
    Returns: URL to screenshot PNG
    """
    response = requests.get(
        f"https://api.neoreader.dev/{url}",
        headers={
            "X-API-Key": "YOUR_API_KEY",
            "X-Respond-With": "pageshot" if full_page else "screenshot"
        },
        allow_redirects=False  # Get redirect URL
    )

    # Returns 302 redirect to screenshot
    screenshot_url = response.headers.get("Location")
    return screenshot_url

# Capture viewport (1024x1024)
viewport_url = capture_screenshot("https://example.com")

# Capture full page
fullpage_url = capture_screenshot("https://example.com", full_page=True)

# Download screenshot
screenshot_response = requests.get(viewport_url)
with open("screenshot.png", "wb") as f:
    f.write(screenshot_response.content)

async function captureForTesting(url, testName) {
  const response = await fetch(
    `https://api.neoreader.dev/${url}`,
    {
      headers: {
        'X-API-Key': 'YOUR_API_KEY',
        'X-Respond-With': 'screenshot'
      },
      redirect: 'manual'  // Get redirect URL
    }
  );

  const screenshotUrl = response.headers.get('location');

  // Download screenshot
  const imageResponse = await fetch(screenshotUrl);
  const buffer = await imageResponse.arrayBuffer();

  // Save for comparison
  await fs.writeFile(
    `./screenshots/${testName}.png`,
    Buffer.from(buffer)
  );

  return screenshotUrl;
}

// Visual regression testing
await captureForTesting('https://app.com', 'homepage-baseline');
await captureForTesting('https://app.com/dashboard', 'dashboard-baseline');

import requests

def scrape_authenticated(url: str, session_cookie: str) -> str:
    """
    Scrape content behind authentication
    Useful for private dashboards, paywalled content
    """
    response = requests.get(
        f"https://api.neoreader.dev/{url}",
        headers={
            "X-API-Key": "YOUR_API_KEY",
            "X-Respond-With": "markdown",
            "X-Set-Cookie": f"session={session_cookie}; user_authenticated=true",
            "X-Target-Selector": ".premium-content"
        }
    )

    return response.text

# Example: Scrape subscriber-only content
cookie = "your_session_cookie_here"
premium_content = scrape_authenticated(
    "https://premium.example.com/article",
    cookie
)

# Works with complex cookie strings
multi_cookie = "session=abc123; _ga=GA1.2.123; pref=dark"
dashboard_content = scrape_authenticated(
    "https://app.example.com/dashboard",
    multi_cookie
)

import requests

def scrape_with_proxy(url: str, proxy_url: str) -> str:
    """
    Scrape content through proxy
    Supports: HTTP, HTTPS, SOCKS5
    """
    response = requests.get(
        f"https://api.neoreader.dev/{url}",
        headers={
            "X-API-Key": "YOUR_API_KEY",
            "X-Respond-With": "markdown",
            "X-Proxy-Url": proxy_url,
            "X-User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
        }
    )

    return response.text

# HTTP proxy
content = scrape_with_proxy(
    "https://us-only-site.com",
    "http://us-proxy.example.com:8080"
)

# SOCKS5 proxy with authentication
content = scrape_with_proxy(
    "https://geo-restricted.com",
    "socks5://user:pass@proxy.example.com:1080"
)

# Rotate proxies for high-volume scraping
proxies = [
    "http://proxy1.example.com:8080",
    "http://proxy2.example.com:8080",
    "http://proxy3.example.com:8080"
]

for i, url in enumerate(urls):
    proxy = proxies[i % len(proxies)]
    content = scrape_with_proxy(url, proxy)