web.py
"""
Web Readers: Website, YouTube, ArXiv, Firecrawl
=================================================
Readers for web-based content sources.
Supported web sources:
- WebsiteReader: Crawls web pages and extracts content
- YouTubeReader: Extracts transcripts from YouTube videos
- ArxivReader: Fetches academic papers from ArXiv
- FirecrawlReader: Advanced web scraping via Firecrawl API
See also: 01_documents.py for PDF/DOCX, 02_data.py for CSV/JSON.
"""
import asyncio
from agno.agent import Agent
from agno.knowledge.embedder.openai import OpenAIEmbedder
from agno.knowledge.knowledge import Knowledge
from agno.knowledge.reader.website_reader import WebsiteReader
from agno.models.openai import OpenAIResponses
from agno.vectordb.qdrant import Qdrant
from agno.vectordb.search import SearchType
# ---------------------------------------------------------------------------
# Setup
# ---------------------------------------------------------------------------
qdrant_url = "http://localhost:6333"
knowledge = Knowledge(
vector_db=Qdrant(
collection="web_readers",
url=qdrant_url,
search_type=SearchType.hybrid,
embedder=OpenAIEmbedder(id="text-embedding-3-small"),
),
)
agent = Agent(
model=OpenAIResponses(id="gpt-5.2"),
knowledge=knowledge,
search_knowledge=True,
markdown=True,
)
# ---------------------------------------------------------------------------
# Run Demo
# ---------------------------------------------------------------------------
if __name__ == "__main__":
async def main():
# --- Website: crawl and extract content ---
print("\n" + "=" * 60)
print("READER: Website (crawl and extract)")
print("=" * 60 + "\n")
# WebsiteReader crawls pages up to max_depth and max_links
website_reader = WebsiteReader(max_depth=1, max_links=5)
await knowledge.ainsert(
name="Agno Docs",
url="https://docs.agno.com/introduction",
reader=website_reader,
)
agent.print_response("What is Agno?", stream=True)
# --- URL: direct URL loading (auto-detected) ---
print("\n" + "=" * 60)
print("READER: Direct URL (auto-detected)")
print("=" * 60 + "\n")
# URLs ending in .pdf, .md, .txt etc. are auto-detected
await knowledge.ainsert(
name="Recipes",
url="https://agno-public.s3.amazonaws.com/recipes/ThaiRecipes.pdf",
)
agent.print_response("What Thai recipes are available?", stream=True)
asyncio.run(main())
Run the Example
1
Set up your virtual environment
uv venv --python 3.12
source .venv/bin/activate
uv venv --python 3.12
.venv\Scripts\activate
2
Install dependencies
uv pip install -U agno beautifulsoup4 fastembed openai pypdf qdrant-client
3
Export your OpenAI API key
export OPENAI_API_KEY="your_openai_api_key_here"
$Env:OPENAI_API_KEY="your_openai_api_key_here"
4
Run Qdrant
docker run -d --name qdrant -p 6333:6333 qdrant/qdrant:latest
5
Run the example
Save the code above as
web.py, then run:python web.py