Convert Any Webpage to Clean Markdown

import requests

def fetch_for_rag(url: str) -> dict:
    response = requests.get(
        f"https://api.neoreader.dev/{url}",
        headers={
            "X-API-Key": "YOUR_API_KEY",
            "X-Respond-With": "markdown",
            "X-Target-Selector": "article, main"
        }
    )
    return {
        "content": response.text,
        "source": url
    }

# Feed directly to vector database
content = fetch_for_rag("https://docs.example.com")
vector_db.upsert(content)

import asyncio
import aiohttp

async def fetch_articles(urls: list[str]):
    async with aiohttp.ClientSession() as session:
        tasks = []
        for url in urls:
            task = session.get(
                f"https://api.neoreader.dev/{url}",
                headers={
                    "X-API-Key": "YOUR_API_KEY",
                    "X-Respond-With": "markdown"
                }
            )
            tasks.append(task)

        responses = await asyncio.gather(*tasks)
        return [await r.text() for r in responses]

from langchain.document_loaders.base import BaseLoader
from langchain.schema import Document
import requests

class NeoReaderLoader(BaseLoader):
    def __init__(self, urls: list[str], api_key: str):
        self.urls = urls
        self.api_key = api_key

    def load(self) -> list[Document]:
        documents = []
        for url in self.urls:
            response = requests.get(
                f"https://api.neoreader.dev/{url}",
                headers={
                    "X-API-Key": self.api_key
                }
            )
            documents.append(Document(
                page_content=response.text,
                metadata={"source": url}
            ))
        return documents