Initial FastAPI CrewAI setup

rag/automation_metadata.py

@@ -1,7 +1,13 @@
 # rag/automation_metadata.py
 from typing import Dict, Any, List, Optional
 
-from core.books.storage import fetch_pending_raw_docs, mark_raw_status, upsert_document_metadata
+from core.books.storage import fetch_pending
+
+
+
+
+
+_raw_docs, mark_raw_status, upsert_document_metadata
 from agents.books.apa_agent import run_metadata_agent  # نفس اللي عندك
 from schemas.books.sources_schema import SourceForAgent, DocMetadata
 

requirements.txt

@@ -10,10 +10,10 @@ crewai[google-genai]
 langchain
 langchain_core
 langchain-community
-crawlee==1.1.0
+# crawlee==1.1.0
 fitz
 apify-fingerprint-datapoints
-browserforge
+# browserforge
 frontend
 parsel
 mistralai
@@ -22,4 +22,6 @@ sentence-transformers
 qdrant-client 
 pymupdf 
 rapidfuzz
-supabase
+supabase
+crawlee==0.3.6
+browserforge==1.1.2

tools/__init__.py

@@ -2,13 +2,17 @@
 # from .scraper import web_scraping_tool
 # from .tavily import search_engine_tool, is_recent
 # from .scraper.scraper_crawlee import WebScrapingCrawleeTool
-from .scraper.scraper_bs4 import WebScrapingToolBS4
-from .scraper.scraper_built_in import pdf_tool, scraping_tool
-from .validate_url import URLValidatorTool
-from .scraper.no_agent.pdf_extractor import extract_pdf_content
-from .scraper.no_agent.bs4_scraper import scrape_with_bs4
-from .scraper.no_agent.crawlee_parse_scraper import crawl_parse_url
-from .scraper.no_agent.crawlee_bs_scraper import crawl_bs_url
-from .searching_scraping_queries import scrape_course, SerperExhaustedError,CURRENT_SERPER_INDEX
+# from .scraper.scraper_bs4 import WebScrapingToolBS4
+# from .scraper.scraper_built_in import pdf_tool, scraping_tool
+# from .validate_url import URLValidatorTool
+# from .scraper.no_agent.pdf_extractor import extract_pdf_content
+# from .scraper.no_agent.bs4_scraper import scrape_with_bs4
+# from .scraper.no_agent.crawlee_parse_scraper import crawl_parse_url
+# from .scraper.no_agent.crawlee_bs_scraper import crawl_bs_url
+from .searching_scraping_queries import (
+    scrape_course,
+    SerperExhaustedError,
+    CURRENT_SERPER_INDEX,
+)
 from .quiz_runner import generate_quiz_for_course
-from .ocr__units_service import extract_text_from_pdf
+from .ocr__units_service import extract_text_from_pdf