Upload 36 files

api/main.py

@@ -1,7 +1,7 @@
 import os
 from fastapi import FastAPI, HTTPException, Query
 from api.schemas import SearchResponse, StatsResponse, DocumentResponse, CrawlRequest
-from api.search import rank_search, wildcard_search
+from api.search import rank_search, wildcard_search, fetch_ddgs_results
 from db.supabase_client import get_supabase
 from indexer.tasks import celery_app
 from typing import Optional
@@ -11,10 +11,13 @@ app = FastAPI(title="Information Retrieval System API")
 @app.get("/search", response_model=SearchResponse)
 async def search(q: str, k: int = Query(10, gt=0)):
     results = rank_search(q, k)
+    ddgs_results, ddgs_images = fetch_ddgs_results(q, 5)
     return {
         "query": q,
         "total_results": len(results),
-        "results": results
+        "results": results,
+        "ddgs_results": ddgs_results,
+        "ddgs_images": ddgs_images
     }
 
 @app.get("/search/wildcard", response_model=SearchResponse)
@@ -25,10 +28,13 @@ async def search_wildcard(q: str, k: int = Query(10, gt=0)):
         return await search(q, k)
         
     results = wildcard_search(q, k)
+    ddgs_results, ddgs_images = fetch_ddgs_results(q, 5)
     return {
         "query": q,
         "total_results": len(results),
-        "results": results
+        "results": results,
+        "ddgs_results": ddgs_results,
+        "ddgs_images": ddgs_images
     }
 
 @app.post("/crawl")

api/schemas.py

@@ -9,10 +9,23 @@ class SearchResult(BaseModel):
     image_url: Optional[str]
     score: float
 
+class DDGSResult(BaseModel):
+    title: str
+    href: str
+    body: str
+
+class DDGSImage(BaseModel):
+    title: str
+    image: str
+    thumbnail: str
+    url: str
+
 class SearchResponse(BaseModel):
     query: str
     total_results: int
     results: List[SearchResult]
+    ddgs_results: Optional[List[DDGSResult]] = []
+    ddgs_images: Optional[List[DDGSImage]] = []
 
 class CrawlRequest(BaseModel):
     url: Optional[str] = None

api/search.py

@@ -1,5 +1,25 @@
 from db.supabase_client import get_supabase
 from indexer.preprocess import preprocess
+from ddgs import DDGS
+
+def fetch_ddgs_results(query: str, max_results: int = 5):
+    """Fetch results and images from DuckDuckGo using ddgs."""
+    results = []
+    images = []
+    
+    try:
+        with DDGS() as ddgs:
+            # Fetch text results
+            ddgs_gen = ddgs.text(query, max_results=max_results, safesearch='on')
+            results = list(ddgs_gen)
+            
+            # Fetch image results
+            ddgs_images_gen = ddgs.images(query, max_results=max_results, safesearch='on')
+            images = list(ddgs_images_gen)
+    except Exception as e:
+        print(f"Error fetching DDGS results: {e}")
+        
+    return results, images
 
 def rank_search(query: str, k: int = 10):
     supabase = get_supabase()

requirements.txt

@@ -1,14 +1,15 @@
-scrapy
-beautifulsoup4
-lxml
-langdetect
-fastapi
-uvicorn
-supabase
-nltk
-PyStemmer
-celery[redis]
-python-dotenv
-httpx
-pydantic-settings
-pydantic
+scrapy
+beautifulsoup4
+lxml
+langdetect
+fastapi
+uvicorn
+supabase
+nltk
+PyStemmer
+celery[redis]
+python-dotenv
+httpx
+pydantic-settings
+pydantic
+ddgs