diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..78c007f59bd5dc8470714b1e5ff1fddc2ed09479 --- /dev/null +++ b/.gitignore @@ -0,0 +1,54 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python + +# Virtual environments +.venv/ +venv/ +ENV/ +env/ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +.tox/ + +# Type checking +.mypy_cache/ +.dmypy.json +dmypy.json + +# Linting +.ruff_cache/ + +# IDEs +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Environment variables +.env +.env.local + +# Logs +*.log + +# Test artifacts +test_*.json +test_*.db +*.sqlite + +# OS +.DS_Store +Thumbs.db + +# Build artifacts +dist/ +build/ +*.egg-info/ diff --git a/app.py b/app.py index 10694e9e7dee7d2d799b3bc656edf71d5c4949de..d534391e720911fefd2447480550c94ccea304a7 100644 --- a/app.py +++ b/app.py @@ -763,13 +763,14 @@ def create_ui() -> gr.Blocks: return app +# Create the demo for HuggingFace Spaces +demo = create_ui() + + def main() -> None: - """Launch the Gradio UI.""" - ui = create_ui() - # Use 127.0.0.1 for local development on Windows for better compatibility - host = "127.0.0.1" if settings.gradio_host == "0.0.0.0" else settings.gradio_host - ui.launch( - server_name=host, + """Launch the Gradio UI locally.""" + demo.launch( + server_name=settings.gradio_host, server_port=settings.gradio_port, share=False, theme=gr.themes.Soft(), diff --git a/requirements.txt b/requirements.txt index 769820836277a0107ee526c7ea5b21ba3cc2774d..8491e0c29bbd812063e6df20f9a750c47fb6b3fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,22 @@ -langgraph>=0.2.0 -langchain-core>=0.3.0 -langchain-openai>=0.2.0 -openai>=1.0.0 -pydantic>=2.0.0 -pydantic-settings>=2.0.0 -gradio>=5.0.0 -praw>=7.7.0 -requests>=2.31.0 -python-dotenv>=1.0.0 -loguru>=0.7.0 -diskcache>=5.6.0 -tenacity>=8.2.0 -langgraph-checkpoint>=4.0.3 -fastapi>=0.136.1 -uvicorn>=0.46.0 -sse-starlette>=3.4.1 +# LangGraph and LangChain - compatible ranges +langgraph>=1.1.0,<2.0.0 +langchain-core>=1.3.0,<2.0.0 +langchain-openai>=1.2.0,<2.0.0 +langgraph-checkpoint>=4.0.0,<5.0.0 + +# OpenAI - transitive dependency via langchain-openai +openai>=2.26.0,<3.0.0 + +# Pydantic +pydantic>=2.10.0,<3.0.0 +pydantic-settings>=2.7.0,<3.0.0 + +# Gradio - match SDK version in README.md +gradio==6.14.0 + +# Utilities +requests>=2.32.0,<3.0.0 +diskcache>=5.6.0,<6.0.0 + +# Note: praw (Reddit API) removed - reddit_scraper.py is not used in the pipeline +# ScrapedComment dataclass is still imported by other scrapers but doesn't require praw diff --git a/src/__pycache__/__init__.cpython-311.pyc b/src/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index df174f2e1089b84587ce9fa640dd861abba6949b..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/src/__pycache__/__init__.cpython-312.pyc b/src/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 08f2ffbe27c0756039f0944d1c0f1b5670d53f3b..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/__pycache__/__init__.cpython-313.pyc b/src/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 7c4105c520697d183218c526f22a5622c593c0fe..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/src/__pycache__/config.cpython-311.pyc b/src/__pycache__/config.cpython-311.pyc deleted file mode 100644 index 1f78e58bdfcaa23096b85de0b5c6067a52a9bf0c..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/config.cpython-311.pyc and /dev/null differ diff --git a/src/__pycache__/config.cpython-312.pyc b/src/__pycache__/config.cpython-312.pyc deleted file mode 100644 index 25b98c74eb6bb30bfb4aeaa9d9631071fd54fe7f..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/config.cpython-312.pyc and /dev/null differ diff --git a/src/__pycache__/config.cpython-313.pyc b/src/__pycache__/config.cpython-313.pyc deleted file mode 100644 index 63f31d5841225bd3d53e0e56a885193060c8f11d..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/config.cpython-313.pyc and /dev/null differ diff --git a/src/__pycache__/graph.cpython-311.pyc b/src/__pycache__/graph.cpython-311.pyc deleted file mode 100644 index e99640818cc693ea91f68f4a16aaae555d965b88..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/graph.cpython-311.pyc and /dev/null differ diff --git a/src/__pycache__/graph.cpython-312.pyc b/src/__pycache__/graph.cpython-312.pyc deleted file mode 100644 index ff19f2381496804a40f24b57af7a66ba1330ab9d..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/graph.cpython-312.pyc and /dev/null differ diff --git a/src/__pycache__/graph.cpython-313.pyc b/src/__pycache__/graph.cpython-313.pyc deleted file mode 100644 index 03b24067b7cb911a26adbe375c7f9c058168f1e0..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/graph.cpython-313.pyc and /dev/null differ diff --git a/src/__pycache__/main.cpython-311.pyc b/src/__pycache__/main.cpython-311.pyc deleted file mode 100644 index 145dff97507a8466743e6344b990f5508ba6e32b..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/main.cpython-311.pyc and /dev/null differ diff --git a/src/__pycache__/main.cpython-312.pyc b/src/__pycache__/main.cpython-312.pyc deleted file mode 100644 index 28b6f35fd2ee166113a9095696212d5876bb6cb8..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/main.cpython-312.pyc and /dev/null differ diff --git a/src/__pycache__/main.cpython-313.pyc b/src/__pycache__/main.cpython-313.pyc deleted file mode 100644 index a820192ff76377eed396228bd53badf99c66ea35..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/main.cpython-313.pyc and /dev/null differ diff --git a/src/__pycache__/run_controller.cpython-311.pyc b/src/__pycache__/run_controller.cpython-311.pyc deleted file mode 100644 index a9097e8fea54b74144c7c5036840367069872217..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/run_controller.cpython-311.pyc and /dev/null differ diff --git a/src/__pycache__/run_controller.cpython-312.pyc b/src/__pycache__/run_controller.cpython-312.pyc deleted file mode 100644 index 8c5a73645a66554ee25d6c717a879a7f6e232ee3..0000000000000000000000000000000000000000 Binary files a/src/__pycache__/run_controller.cpython-312.pyc and /dev/null differ diff --git a/src/agents/__pycache__/__init__.cpython-311.pyc b/src/agents/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index 6d7751858b20e9dbb063827909808bd81b990c87..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/src/agents/__pycache__/__init__.cpython-312.pyc b/src/agents/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 9d8587215432fd3b02c5d2409474d7f97f94ccef..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/agents/__pycache__/__init__.cpython-313.pyc b/src/agents/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 89962e31e5e11595c31727e189e2163dd4e55af0..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/src/agents/__pycache__/critic.cpython-311.pyc b/src/agents/__pycache__/critic.cpython-311.pyc deleted file mode 100644 index 73bdfc389352f564d1123c23f7f13513243a3311..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/critic.cpython-311.pyc and /dev/null differ diff --git a/src/agents/__pycache__/critic.cpython-312.pyc b/src/agents/__pycache__/critic.cpython-312.pyc deleted file mode 100644 index 296109c61ddbb22dd995188e4c5ec175c776b7f6..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/critic.cpython-312.pyc and /dev/null differ diff --git a/src/agents/__pycache__/critic.cpython-313.pyc b/src/agents/__pycache__/critic.cpython-313.pyc deleted file mode 100644 index a0947dae862605b8f42a245d2b915e1819aeeaff..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/critic.cpython-313.pyc and /dev/null differ diff --git a/src/agents/__pycache__/idea_generator.cpython-311.pyc b/src/agents/__pycache__/idea_generator.cpython-311.pyc deleted file mode 100644 index 25d53dfac64dbddd293abc0c9f388baa2c5a1dde..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/idea_generator.cpython-311.pyc and /dev/null differ diff --git a/src/agents/__pycache__/idea_generator.cpython-312.pyc b/src/agents/__pycache__/idea_generator.cpython-312.pyc deleted file mode 100644 index 184e97daf2384d3704efa9d01922f58c6ef9956c..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/idea_generator.cpython-312.pyc and /dev/null differ diff --git a/src/agents/__pycache__/idea_generator.cpython-313.pyc b/src/agents/__pycache__/idea_generator.cpython-313.pyc deleted file mode 100644 index 89f2bde2180d02fe77893c631e041416bc79e4a2..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/idea_generator.cpython-313.pyc and /dev/null differ diff --git a/src/agents/__pycache__/orchestrator.cpython-311.pyc b/src/agents/__pycache__/orchestrator.cpython-311.pyc deleted file mode 100644 index e00a97d58809ac93dc9f63fb0be3637b6699d3b1..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/orchestrator.cpython-311.pyc and /dev/null differ diff --git a/src/agents/__pycache__/orchestrator.cpython-312.pyc b/src/agents/__pycache__/orchestrator.cpython-312.pyc deleted file mode 100644 index 213b3ca64f6106420c1930e1788f9746bfe14cc8..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/orchestrator.cpython-312.pyc and /dev/null differ diff --git a/src/agents/__pycache__/orchestrator.cpython-313.pyc b/src/agents/__pycache__/orchestrator.cpython-313.pyc deleted file mode 100644 index 4c120fafdcf4ce8c06f4d763f95ba33451b81b7b..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/orchestrator.cpython-313.pyc and /dev/null differ diff --git a/src/agents/__pycache__/pain_point_miner.cpython-311.pyc b/src/agents/__pycache__/pain_point_miner.cpython-311.pyc deleted file mode 100644 index 58f71b435f9226f5d03cad8fb59b838fe4a1572d..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/pain_point_miner.cpython-311.pyc and /dev/null differ diff --git a/src/agents/__pycache__/pain_point_miner.cpython-312.pyc b/src/agents/__pycache__/pain_point_miner.cpython-312.pyc deleted file mode 100644 index 2d3e9477ea6ba3de070e45dbcf63738de758cccd..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/pain_point_miner.cpython-312.pyc and /dev/null differ diff --git a/src/agents/__pycache__/pain_point_miner.cpython-313.pyc b/src/agents/__pycache__/pain_point_miner.cpython-313.pyc deleted file mode 100644 index ff9af6bdcfab6bffd12376db186bf19c35ff548e..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/pain_point_miner.cpython-313.pyc and /dev/null differ diff --git a/src/agents/__pycache__/pitch_writer.cpython-311.pyc b/src/agents/__pycache__/pitch_writer.cpython-311.pyc deleted file mode 100644 index 8d6a42a9221dd505b2195bca7d0d0f9ca3d2ebc3..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/pitch_writer.cpython-311.pyc and /dev/null differ diff --git a/src/agents/__pycache__/pitch_writer.cpython-312.pyc b/src/agents/__pycache__/pitch_writer.cpython-312.pyc deleted file mode 100644 index 646cdf347eb8eb998443d7c03bf5ac6c4df81ecb..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/pitch_writer.cpython-312.pyc and /dev/null differ diff --git a/src/agents/__pycache__/pitch_writer.cpython-313.pyc b/src/agents/__pycache__/pitch_writer.cpython-313.pyc deleted file mode 100644 index ab5ec18dfb7946c518d04f5de67c6d274f07cd7f..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/pitch_writer.cpython-313.pyc and /dev/null differ diff --git a/src/agents/__pycache__/scorer.cpython-311.pyc b/src/agents/__pycache__/scorer.cpython-311.pyc deleted file mode 100644 index 415717b709594538fad8f769fb3580d8df19fc42..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/scorer.cpython-311.pyc and /dev/null differ diff --git a/src/agents/__pycache__/scorer.cpython-312.pyc b/src/agents/__pycache__/scorer.cpython-312.pyc deleted file mode 100644 index 1b76fb5bc0c3ef4a4ebe4fc8defb9240cb3c7e6c..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/scorer.cpython-312.pyc and /dev/null differ diff --git a/src/agents/__pycache__/scorer.cpython-313.pyc b/src/agents/__pycache__/scorer.cpython-313.pyc deleted file mode 100644 index 7a921eefc513fa942dbddd1a99436af50a5b3b85..0000000000000000000000000000000000000000 Binary files a/src/agents/__pycache__/scorer.cpython-313.pyc and /dev/null differ diff --git a/src/agents/pain_point_miner.py b/src/agents/pain_point_miner.py index d36ddb12d409dc9de5a461db4f636fea8cbd15ea..2c73c7d65a8c16cd441a54dab4dcd96473ddc77f 100644 --- a/src/agents/pain_point_miner.py +++ b/src/agents/pain_point_miner.py @@ -22,7 +22,8 @@ from src.config import settings from src.llm.client import coerce_rubric_bools, coerce_yes_no, extract_json, get_llm from src.llm.prompts import get_prompt from src.state.schema import DataSource, PainPoint, PainPointRubric, PipelineStage, VentureForgeState -from src.tools.reddit_scraper import COMMUNITY_MAP, ScrapedComment, validate_quote +from src.tools.types import ScrapedComment +from src.tools.reddit_scraper import COMMUNITY_MAP, validate_quote from src.tools.tavily_fallback import search_communities from src.tools.hackernews_scraper import scrape_for_domain as hn_scrape_for_domain from src.tools.producthunt_scraper import scrape_for_domain as ph_scrape_for_domain diff --git a/src/llm/__pycache__/__init__.cpython-311.pyc b/src/llm/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index 63c23e42b8e700c39bd596b39b163e148fc6d3a3..0000000000000000000000000000000000000000 Binary files a/src/llm/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/src/llm/__pycache__/__init__.cpython-312.pyc b/src/llm/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 46bad7963dac370f8938bc8b48a82ce65cb4785b..0000000000000000000000000000000000000000 Binary files a/src/llm/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/llm/__pycache__/__init__.cpython-313.pyc b/src/llm/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 43c6665d3537823185c261a63ce25de359dc79f3..0000000000000000000000000000000000000000 Binary files a/src/llm/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/src/llm/__pycache__/client.cpython-311.pyc b/src/llm/__pycache__/client.cpython-311.pyc deleted file mode 100644 index 3127b50f90bb4c4cea1c26ba54e65af4382f7607..0000000000000000000000000000000000000000 Binary files a/src/llm/__pycache__/client.cpython-311.pyc and /dev/null differ diff --git a/src/llm/__pycache__/client.cpython-312.pyc b/src/llm/__pycache__/client.cpython-312.pyc deleted file mode 100644 index c8ce681c58d1611e76061dbb00ade477c0662cb0..0000000000000000000000000000000000000000 Binary files a/src/llm/__pycache__/client.cpython-312.pyc and /dev/null differ diff --git a/src/llm/__pycache__/client.cpython-313.pyc b/src/llm/__pycache__/client.cpython-313.pyc deleted file mode 100644 index 70ddedc85ca7abe8835a6bc0a062013aaf728137..0000000000000000000000000000000000000000 Binary files a/src/llm/__pycache__/client.cpython-313.pyc and /dev/null differ diff --git a/src/llm/__pycache__/prompts.cpython-311.pyc b/src/llm/__pycache__/prompts.cpython-311.pyc deleted file mode 100644 index 328196fcffbd63d0d37fe81947fa235d1fe09f2f..0000000000000000000000000000000000000000 Binary files a/src/llm/__pycache__/prompts.cpython-311.pyc and /dev/null differ diff --git a/src/llm/__pycache__/prompts.cpython-312.pyc b/src/llm/__pycache__/prompts.cpython-312.pyc deleted file mode 100644 index 8fd22117acf1c0a3c1bf811c483be5b76dda8165..0000000000000000000000000000000000000000 Binary files a/src/llm/__pycache__/prompts.cpython-312.pyc and /dev/null differ diff --git a/src/state/__pycache__/__init__.cpython-311.pyc b/src/state/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index b95959962beefd661c136daeb1cba92276b093c9..0000000000000000000000000000000000000000 Binary files a/src/state/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/src/state/__pycache__/__init__.cpython-312.pyc b/src/state/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 00c64ad9072abc1a9a9abda6e6593c8e91d2dfbd..0000000000000000000000000000000000000000 Binary files a/src/state/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/state/__pycache__/__init__.cpython-313.pyc b/src/state/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 4d1260c189922005e4da22eb1848793be30a805c..0000000000000000000000000000000000000000 Binary files a/src/state/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/src/state/__pycache__/schema.cpython-311.pyc b/src/state/__pycache__/schema.cpython-311.pyc deleted file mode 100644 index d0c2552998f361115d3421a190b854d9e8715709..0000000000000000000000000000000000000000 Binary files a/src/state/__pycache__/schema.cpython-311.pyc and /dev/null differ diff --git a/src/state/__pycache__/schema.cpython-312.pyc b/src/state/__pycache__/schema.cpython-312.pyc deleted file mode 100644 index f13d27e262377633bc7c7369d7e6d0129eb15e32..0000000000000000000000000000000000000000 Binary files a/src/state/__pycache__/schema.cpython-312.pyc and /dev/null differ diff --git a/src/state/__pycache__/schema.cpython-313.pyc b/src/state/__pycache__/schema.cpython-313.pyc deleted file mode 100644 index 3cb7e1942ebfc8cb19df44beb1b48ad1e0620693..0000000000000000000000000000000000000000 Binary files a/src/state/__pycache__/schema.cpython-313.pyc and /dev/null differ diff --git a/src/tools/__pycache__/__init__.cpython-311.pyc b/src/tools/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index 0e09e581f2264f63faa9c4b87158d55c79d0b057..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/src/tools/__pycache__/__init__.cpython-312.pyc b/src/tools/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index f34d448819bd6a468bf47cd5331ef766ec44a2df..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/tools/__pycache__/__init__.cpython-313.pyc b/src/tools/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index a147caeca5e28bf53fbde2d01e6e5432f2c7e260..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/src/tools/__pycache__/hackernews_scraper.cpython-311.pyc b/src/tools/__pycache__/hackernews_scraper.cpython-311.pyc deleted file mode 100644 index 07837bed451f42333946368302b7bc3205e45e9c..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/hackernews_scraper.cpython-311.pyc and /dev/null differ diff --git a/src/tools/__pycache__/hackernews_scraper.cpython-312.pyc b/src/tools/__pycache__/hackernews_scraper.cpython-312.pyc deleted file mode 100644 index 7bef6132a0a5ba177ca4d2746f492c4ab296839e..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/hackernews_scraper.cpython-312.pyc and /dev/null differ diff --git a/src/tools/__pycache__/producthunt_scraper.cpython-311.pyc b/src/tools/__pycache__/producthunt_scraper.cpython-311.pyc deleted file mode 100644 index a05db0a579a2b3f8f53f2d6bef31e36792ff4aa0..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/producthunt_scraper.cpython-311.pyc and /dev/null differ diff --git a/src/tools/__pycache__/producthunt_scraper.cpython-312.pyc b/src/tools/__pycache__/producthunt_scraper.cpython-312.pyc deleted file mode 100644 index fbae9610e0b12c95f61e946b65db5b5779350dd4..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/producthunt_scraper.cpython-312.pyc and /dev/null differ diff --git a/src/tools/__pycache__/reddit_scraper.cpython-311.pyc b/src/tools/__pycache__/reddit_scraper.cpython-311.pyc deleted file mode 100644 index 81970e74334c7a6e72b3800f1b5d0cb811c3617e..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/reddit_scraper.cpython-311.pyc and /dev/null differ diff --git a/src/tools/__pycache__/reddit_scraper.cpython-312.pyc b/src/tools/__pycache__/reddit_scraper.cpython-312.pyc deleted file mode 100644 index f5641c45bfc62d61b948d8548e2a52b6fd947e6e..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/reddit_scraper.cpython-312.pyc and /dev/null differ diff --git a/src/tools/__pycache__/tavily_content_scraper.cpython-311.pyc b/src/tools/__pycache__/tavily_content_scraper.cpython-311.pyc deleted file mode 100644 index 220df0f51553a97582366d9ec4dbedd7bfb83c21..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/tavily_content_scraper.cpython-311.pyc and /dev/null differ diff --git a/src/tools/__pycache__/tavily_content_scraper.cpython-312.pyc b/src/tools/__pycache__/tavily_content_scraper.cpython-312.pyc deleted file mode 100644 index 07dfe2b743a4cb7b67b3948826d89b5c13e60868..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/tavily_content_scraper.cpython-312.pyc and /dev/null differ diff --git a/src/tools/__pycache__/tavily_content_scraper.cpython-313.pyc b/src/tools/__pycache__/tavily_content_scraper.cpython-313.pyc deleted file mode 100644 index 69ed73779ae563c1ae6e53d346fc02e1c31fac40..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/tavily_content_scraper.cpython-313.pyc and /dev/null differ diff --git a/src/tools/__pycache__/tavily_fallback.cpython-311.pyc b/src/tools/__pycache__/tavily_fallback.cpython-311.pyc deleted file mode 100644 index b98fd3630034f94451a466741be67ec91cc238fc..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/tavily_fallback.cpython-311.pyc and /dev/null differ diff --git a/src/tools/__pycache__/tavily_fallback.cpython-312.pyc b/src/tools/__pycache__/tavily_fallback.cpython-312.pyc deleted file mode 100644 index b578333a1046297eae9c00b45af0137faab9db8d..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/tavily_fallback.cpython-312.pyc and /dev/null differ diff --git a/src/tools/__pycache__/youtube_scraper.cpython-311.pyc b/src/tools/__pycache__/youtube_scraper.cpython-311.pyc deleted file mode 100644 index e4f6ee3c041f657298bce5157c2ab93d30c8dd1d..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/youtube_scraper.cpython-311.pyc and /dev/null differ diff --git a/src/tools/__pycache__/youtube_scraper.cpython-312.pyc b/src/tools/__pycache__/youtube_scraper.cpython-312.pyc deleted file mode 100644 index d820160d6e9d0d1189c5ec942baadd60343421ce..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/youtube_scraper.cpython-312.pyc and /dev/null differ diff --git a/src/tools/__pycache__/youtube_scraper.cpython-313.pyc b/src/tools/__pycache__/youtube_scraper.cpython-313.pyc deleted file mode 100644 index 4be3f48ff1a3dc0e3a135d023ffd926470a531db..0000000000000000000000000000000000000000 Binary files a/src/tools/__pycache__/youtube_scraper.cpython-313.pyc and /dev/null differ diff --git a/src/tools/hackernews_scraper.py b/src/tools/hackernews_scraper.py index 547636b04011677dd4c41feb91c7b982a4f04487..6b24b80020a2feccd134192e1b602bd3306d5e77 100644 --- a/src/tools/hackernews_scraper.py +++ b/src/tools/hackernews_scraper.py @@ -16,7 +16,7 @@ import diskcache import requests from src.config import settings -from src.tools.reddit_scraper import ScrapedComment +from src.tools.types import ScrapedComment logger = logging.getLogger(__name__) diff --git a/src/tools/producthunt_scraper.py b/src/tools/producthunt_scraper.py index bed7b1749acbb164400ebebfd5c2c52f7118adba..a56206a319ff24c8f01a96b5749843dc89126ba0 100644 --- a/src/tools/producthunt_scraper.py +++ b/src/tools/producthunt_scraper.py @@ -22,7 +22,7 @@ import diskcache import requests from src.config import settings -from src.tools.reddit_scraper import ScrapedComment +from src.tools.types import ScrapedComment logger = logging.getLogger(__name__) diff --git a/src/tools/reddit_scraper.py b/src/tools/reddit_scraper.py index 8821443c292f4bfa2270c7cda12796b2970f2c05..abee1694041d104225335aec6c03ed76ed8e401d 100644 --- a/src/tools/reddit_scraper.py +++ b/src/tools/reddit_scraper.py @@ -18,11 +18,19 @@ from dataclasses import dataclass from typing import Iterator import diskcache -import praw import requests import urllib3 from src.config import settings +from src.tools.types import ScrapedComment + +# Optional import - praw not required if Reddit scraping is disabled +try: + import praw + PRAW_AVAILABLE = True +except ImportError: + PRAW_AVAILABLE = False + praw = None # type: ignore logger = logging.getLogger(__name__) @@ -115,17 +123,6 @@ def resolve_domain(domain: str) -> tuple[str, list[str]]: return "general_other", COMMUNITY_MAP["general_other"] -# ------------------------------------------------------------------ -# Scraped-comment data structure -# ------------------------------------------------------------------ -@dataclass(frozen=True) -class ScrapedComment: - text: str - url: str - subreddit: str - post_title: str - - # ------------------------------------------------------------------ # PRAW Reddit client # ------------------------------------------------------------------ diff --git a/src/tools/tavily_content_scraper.py b/src/tools/tavily_content_scraper.py index f5e98a14a1ee0e4f35e726beb9b73cbdee475cf1..538f91d8b50d3a0de19cdce5ecf2b6ba22be25a6 100644 --- a/src/tools/tavily_content_scraper.py +++ b/src/tools/tavily_content_scraper.py @@ -17,7 +17,7 @@ import diskcache import requests from src.config import settings -from src.tools.reddit_scraper import ScrapedComment +from src.tools.types import ScrapedComment logger = logging.getLogger(__name__) diff --git a/src/tools/types.py b/src/tools/types.py new file mode 100644 index 0000000000000000000000000000000000000000..dec4df76084a82b139c99e168ee2f53cf5031f7c --- /dev/null +++ b/src/tools/types.py @@ -0,0 +1,13 @@ +"""Shared data types for scraper tools.""" +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ScrapedComment: + """Structured comment data from any scraper source.""" + text: str + url: str + subreddit: str # or "source" for non-Reddit scrapers + post_title: str diff --git a/src/tools/youtube_scraper.py b/src/tools/youtube_scraper.py index d13cc06bb27f860b3161b1074dc1c54de2f26f78..122f0c73fade31a99a6738f21643af01fa9b2dee 100644 --- a/src/tools/youtube_scraper.py +++ b/src/tools/youtube_scraper.py @@ -23,7 +23,7 @@ import diskcache import requests from src.config import settings -from src.tools.reddit_scraper import ScrapedComment +from src.tools.types import ScrapedComment logger = logging.getLogger(__name__)