Remove useless endpoint

app.py

@@ -1,8 +1,7 @@
 from contextlib import asynccontextmanager
 from typing import Optional
-import expiringdict
 from fastapi import APIRouter, FastAPI
-from fastapi.routing import APIRouter as Router
+from fastapi.routing import APIRouter
 import httpx
 from pydantic import BaseModel, Field
 from playwright.async_api import async_playwright, Browser, BrowserContext, Page
@@ -10,7 +9,7 @@ import logging
 import uvicorn
 
 from scrap import scrap_patent_async, scrap_patent_bulk_async
-from search import APISearchResults, APIPatentResults, query_bing_search, query_brave_search, query_ddg_search, query_google_patents
+from serp import SerpResults, query_bing_search, query_brave_search, query_ddg_search, query_google_patents
 
 logging.basicConfig(
     level=logging.INFO,
@@ -37,42 +36,31 @@ async def api_lifespan(app: FastAPI):
     await pw_browser.close()
     await playwright.stop()
 
-app = FastAPI(lifespan=api_lifespan)
-backend_status = expiringdict.ExpiringDict(max_len=5, max_age_seconds=15*60)
+app = FastAPI(lifespan=api_lifespan, docs_url="/")
 
 # Router for scrapping related endpoints
 scrap_router = APIRouter(prefix="/scrap", tags=["scrapping"])
-# Router for search related endpoints
-search_router = APIRouter(prefix="/search", tags=["search"])
-
-
-@app.get('/')
-async def status():
-    backend_keys = [k[0] for k in backend_status.items()]
-    backend_status_dict = {}
-
-    for k in backend_keys:
-        backend_status_dict[k] = backend_status.get(k)
-    return {"status": "running", "backend_status": backend_status_dict}
+# Router for SERP-scrapping related endpoints
+serp_router = APIRouter(prefix="/serp", tags=["serp scrapping"])
 
 # ===================== Search endpoints =====================
 
 
-class APISearchParams(BaseModel):
+class SerpQuery(BaseModel):
     queries: list[str] = Field(...,
                                description="The list of queries to search for")
     n_results: int = Field(
         10, description="Number of results to return for each query. Valid values are 10, 25, 50 and 100")
 
 
-@search_router.post("/search_scholar")
-async def query_google_scholar(params: APISearchParams):
+@serp_router.post("/search_scholar")
+async def query_google_scholar(params: SerpQuery):
     """Queries google scholar for the specified query"""
     return {"error": "Unimplemented"}
 
 
-@search_router.post("/search_patents")
-async def search_patents(params: APISearchParams) -> APIPatentResults:
+@serp_router.post("/search_patents")
+async def search_patents(params: SerpQuery) -> SerpResults:
     """Searches google patents for the specified queries and returns the found documents."""
     results = []
     for q in params.queries:
@@ -81,14 +69,13 @@ async def search_patents(params: APISearchParams) -> APIPatentResults:
             res = await query_google_patents(pw_browser, q, params.n_results)
             results.extend(res)
         except Exception as e:
-            backend_status["gpatents"] = "rate-limited"
             logging.error(
                 f"Failed to query Google Patents with query `{q}`: {e}")
-    return APIPatentResults(results=results, error=None)
+    return SerpResults(results=results, error=None)
 
 
-@search_router.post("/search_brave")
-async def search_brave(params: APISearchParams) -> APISearchResults:
+@serp_router.post("/search_brave")
+async def search_brave(params: SerpQuery) -> SerpResults:
     """Searches brave search for the specified queries and returns the found documents."""
     results = []
     last_exception: Optional[Exception] = None
@@ -99,15 +86,14 @@ async def search_brave(params: APISearchParams) -> APISearchResults:
             results.extend(res)
         except Exception as e:
             last_exception = e
-            backend_status["brave"] = "rate-limited"
             logging.error(
                 f"Failed to query Brave search with query `{q}`: {e}")
 
-    return APISearchResults(results=results, error=str(last_exception) if len(results) == 0 and last_exception else None)
+    return SerpResults(results=results, error=str(last_exception) if len(results) == 0 and last_exception else None)
 
 
-@search_router.post("/search_bing")
-async def search_bing(params: APISearchParams) -> APISearchResults:
+@serp_router.post("/search_bing")
+async def search_bing(params: SerpQuery) -> SerpResults:
     """Searches Bing search for the specified queries and returns the found documents."""
     results = []
     last_exception: Optional[Exception] = None
@@ -118,15 +104,14 @@ async def search_bing(params: APISearchParams) -> APISearchResults:
             results.extend(res)
         except Exception as e:
             last_exception = e
-            backend_status["bing"] = "rate-limited"
             logging.error(
                 f"Failed to query Bing search with query `{q}`: {e}")
 
-    return APISearchResults(results=results, error=str(last_exception) if len(results) == 0 and last_exception else None)
+    return SerpResults(results=results, error=str(last_exception) if len(results) == 0 and last_exception else None)
 
 
-@search_router.post("/search_duck")
-async def search_duck(params: APISearchParams) -> APISearchResults:
+@serp_router.post("/search_duck")
+async def search_duck(params: SerpQuery) -> SerpResults:
     """Searches duckduckgo for the specified queries and returns the found documents"""
     results = []
     last_exception: Optional[Exception] = None
@@ -138,15 +123,14 @@ async def search_duck(params: APISearchParams) -> APISearchResults:
             results.extend(res)
         except Exception as e:
             last_exception = e
-            backend_status["duckduckgo"] = "rate-limited"
             logging.error(f"Failed to query DDG with query `{q}`: {e}")
 
-    return APISearchResults(results=results, error=str(last_exception) if len(results) == 0 and last_exception else None)
+    return SerpResults(results=results, error=str(last_exception) if len(results) == 0 and last_exception else None)
 
 
-@search_router.post("/search")
+@serp_router.post("/search")
 @app.post("/search")
-async def search(params: APISearchParams):
+async def search(params: SerpQuery):
     """Attempts to search the specified queries using ALL backends"""
     results = []
 
@@ -180,9 +164,9 @@ async def search(params: APISearchParams):
             logging.info("Trying with next browser backend.")
 
         if len(results) == 0:
-            return APISearchResults(results=[], error="All backends are rate-limited.")
+            return SerpResults(results=[], error="All backends are rate-limited.")
 
-    return APISearchResults(results=results, error=None)
+    return SerpResults(results=results, error=None)
 
 # =========================== Scrapping endpoints ===========================
 
@@ -209,7 +193,7 @@ async def scrap_patents(params: ScrapPatentsRequest):
 
 # ===============================================================================
 
-app.include_router(search_router)
+app.include_router(serp_router)
 app.include_router(scrap_router)
 
 uvicorn.run(app, host="0.0.0.0", port=7860)

requirements.txt

@@ -3,6 +3,5 @@ uvicorn
 pydantic
 playwright
 duckduckgo_search
-expiringdict
 beautifulsoup4
 httpx

search.py → serp.py

@@ -7,14 +7,8 @@ from urllib.parse import quote_plus
 import logging
 import re
 
-
-class APIPatentResults(BaseModel):
-    """Response of /search_patents endpoint"""
-    error: Optional[str]
-    results: Optional[list[dict]]
-
-
-class APISearchResults(BaseModel):
+class SerpResults(BaseModel):
+    """Model for SERP scrapping results"""
     error: Optional[str]
     results: Optional[list[dict]]