Update app.py

app.py

@@ -7,6 +7,9 @@ import asyncio
 from contextlib import asynccontextmanager
 from dotenv import load_dotenv
 from operator import itemgetter
+import requests
+from bs4 import BeautifulSoup
+
 
 from fastapi import FastAPI, Depends, HTTPException, Header
 from fastapi.responses import JSONResponse
@@ -49,7 +52,7 @@ load_dotenv()
 
 vector_cache = {}
 ml_models = {}
-
+secret = ""
 PRELOAD_URLS = [
     "https://hackrx.blob.core.windows.net/assets/Arogya%20Sanjeevani%20Policy%20-%20CIN%20-%20U10200WB1906GOI001713%201.pdf?sv=2023-01-03&st=2025-07-21T08%3A29%3A02Z&se=2025-09-22T08%3A29%3A00Z&sr=b&sp=r&sig=nzrz1K9Iurt%2BBXom%2FB%2BMPTFMFP3PRnIvEsipAX10Ig4%3D",
     "https://hackrx.blob.core.windows.net/assets/Super_Splendor_(Feb_2023).pdf?sv=2023-01-03&st=2025-07-21T08%3A10%3A00Z&se=2025-09-22T08%3A10%3A00Z&sr=b&sp=r&sig=vhHrl63YtrEOCsAy%2BpVKr20b3ZUo5HMz1lF9%2BJh6LQ0%3D",
@@ -173,6 +176,17 @@ Step 3 – **Final Output**:
 # We pass the lifespan function to the FastAPI constructor
 app = FastAPI(title="HackRX RAG Server", lifespan=lifespan)
 
+def store_secret(url: str):
+    url_c = url
+    r = requests.get(url_c)
+    r.raise_for_status()
+    soup = BeautifulSoup(r.text, "html.parser")
+    token = (soup.find(id="token") or soup).get_text(strip=True)
+    m = re.search(r"[0-9a-fA-F]{64}", token)
+    token = m.group(0) if m else token
+    secret = token
+
+
 # --- 3. API Key Verification ---
 TEAM_API_KEY = os.getenv("TEAM_API_KEY")
 TEAM_API_KEY2 = os.getenv("TEAM_API_KEY2")
@@ -181,7 +195,7 @@ def verify_api_key(authorization: str = Header(...)):
     if not authorization.startswith("Bearer "):
         raise HTTPException(status_code=401, detail="Invalid Authorization header format")
     token = authorization.split("Bearer ")[1]
-    if token != TEAM_API_KEY and token != TEAM_API_KEY2:
+    if token != TEAM_API_KEY and token != TEAM_API_KEY2 and token != secret:
         raise HTTPException(status_code=403, detail="Invalid or missing API key")
 
 
@@ -214,72 +228,76 @@ def parse_llm_response(content: str) -> str:
 # --- 5. Main API Endpoint ---
 @app.post("/api/v1/hackrx/run", response_model=RunResponse, dependencies=[Depends(verify_api_key)])
 async def run_hackrx(req: RunRequest):
-    doc_url = str(req.documents) 
-    start_time = time.time()
-    # if(doc_url not in vector_cache):
-    chunks = load_and_chunk(str(req.documents))
-    if not chunks:
-        return JSONResponse({"error": "No documents could be processed."}, status_code=400)    
-    end_time = time.time() - start_time
-    print(f"chunking done: {end_time}")
-    # if not chunks:
-    #     return JSONResponse({"error": "No documents could be processed."}, status_code=400)
-
-
-    start_time2 = time.time()
-    # ✅ Reuse vectorstore if already cached
-    if doc_url in vector_cache:
-        print(f"♻ Using cached vectorstore for: {doc_url}")
-        vectorstore = vector_cache[doc_url]
-    else:
-        print(f"📄 Processing new document: {doc_url}")
-        # Build vectorstore & save to cache
-        vectorstore = await FAISS.afrom_documents(documents=chunks, embedding=ml_models["embedder"])
-        vector_cache[doc_url] = vectorstore  # store in memory cache
-        print(f"✅ Vectorstore cached for: {doc_url}")
-    end_time2 = time.time() - start_time2
-    print(f"vector done: {end_time2}")
-
-    # start_time2 = time.time()
-    # vectorstore = await FAISS.afrom_documents(
-    #     documents=chunks, 
-    #     embedding=ml_models["embedder"]
-    # )
-    # end_time2 = time.time() - start_time2
-    # print(f"vector done: {end_time2}")
-    # dense_retriever = vectorstore.as_retriever(search_type="mmr",search_kwargs={"k": 8})
-    dense_retriever = vectorstore.as_retriever(search_type="mmr",search_kwargs={"k": 14 ,"lambda_mult": 0.7} ) # prev 16
-    # dense_retriever = vectorstore.as_retriever(search_type="similarity" ,search_kwargs={"k": 11} )
+    doc_url = str(req.documents)
+    lower_url = doc_url.lower()
+    if "get-secret-token" in lower_url:
+        store_secret(doc_url)
+    else:       
+        start_time = time.time()
+        # if(doc_url not in vector_cache):
+        chunks = load_and_chunk(str(req.documents))
+        if not chunks:
+            return JSONResponse({"error": "No documents could be processed."}, status_code=400)    
+        end_time = time.time() - start_time
+        print(f"chunking done: {end_time}")
+        # if not chunks:
+        #     return JSONResponse({"error": "No documents could be processed."}, status_code=400)
+       
+       
+        start_time2 = time.time()
+        # ✅ Reuse vectorstore if already cached
+        if doc_url in vector_cache:
+            print(f"♻ Using cached vectorstore for: {doc_url}")
+            vectorstore = vector_cache[doc_url]
+        else:
+            print(f"📄 Processing new document: {doc_url}")
+            # Build vectorstore & save to cache
+            vectorstore = await FAISS.afrom_documents(documents=chunks, embedding=ml_models["embedder"])
+            vector_cache[doc_url] = vectorstore  # store in memory cache
+            print(f"✅ Vectorstore cached for: {doc_url}")
+        end_time2 = time.time() - start_time2
+        print(f"vector done: {end_time2}")
+
+        # start_time2 = time.time()
+        # vectorstore = await FAISS.afrom_documents(
+        #     documents=chunks, 
+        #     embedding=ml_models["embedder"]
+        # )
+        # end_time2 = time.time() - start_time2
+        # print(f"vector done: {end_time2}")
+        # dense_retriever = vectorstore.as_retriever(search_type="mmr",search_kwargs={"k": 8})
+        dense_retriever = vectorstore.as_retriever(search_type="mmr",search_kwargs={"k": 14 ,"lambda_mult": 0.7} ) # prev 16
+        # dense_retriever = vectorstore.as_retriever(search_type="similarity" ,search_kwargs={"k": 11} )
+        
+     
+        # Create retrievers using the pre-loaded models from our ml_models dictionary
+        keyword_retriever = BM25Retriever.from_documents(chunks)
+        keyword_retriever.k = 9 #prev 11
+        # dense_retriever = Chroma.from_documents(documents=chunks, embedding=ml_models["embedder"]).as_retriever()
+        ensemble_retriever = EnsembleRetriever(retrievers=[keyword_retriever, dense_retriever], weights=[0.3, 0.7],search_kwargs={"k": 14}) #prev 16
+        ### to make it faster we are now using our built reranker thats why commenting the code below
+        # compression_retriever = ContextualCompressionRetriever(
+        #     base_retriever=ensemble_retriever, base_compressor=ml_models["reranker_compressor"]
+        # )
+     
+        # Define the RAG chain using pre-loaded components
+        hybrid_rag_chain = (
+            {"context": itemgetter("full_query") | ensemble_retriever, "full_query": itemgetter("full_query")}
+            | ml_models["prompt_template"]
+            | ml_models["llm"]
+        )
+     
+        tasks = [hybrid_rag_chain.ainvoke({"full_query": q}) for q in req.questions]
+        results = await asyncio.gather(*tasks)
+        # answers = [parse_llm_response(result.content) for result in results]
+        answers = []
+
+        for msg in results:
+            # Safely access the content field
+            if hasattr(msg, "content"):
+                answers.append(msg.content.strip())    
     
-
-    # Create retrievers using the pre-loaded models from our ml_models dictionary
-    keyword_retriever = BM25Retriever.from_documents(chunks)
-    keyword_retriever.k = 9 #prev 11
-    # dense_retriever = Chroma.from_documents(documents=chunks, embedding=ml_models["embedder"]).as_retriever()
-    ensemble_retriever = EnsembleRetriever(retrievers=[keyword_retriever, dense_retriever], weights=[0.3, 0.7],search_kwargs={"k": 14}) #prev 16
-    ### to make it faster we are now using our built reranker thats why commenting the code below
-    # compression_retriever = ContextualCompressionRetriever(
-    #     base_retriever=ensemble_retriever, base_compressor=ml_models["reranker_compressor"]
-    # )
-
-    # Define the RAG chain using pre-loaded components
-    hybrid_rag_chain = (
-        {"context": itemgetter("full_query") | ensemble_retriever, "full_query": itemgetter("full_query")}
-        | ml_models["prompt_template"]
-        | ml_models["llm"]
-    )
-
-    tasks = [hybrid_rag_chain.ainvoke({"full_query": q}) for q in req.questions]
-    results = await asyncio.gather(*tasks)
-    # answers = [parse_llm_response(result.content) for result in results]
-    answers = []
-
-    for msg in results:
-        # Safely access the content field
-        if hasattr(msg, "content"):
-            answers.append(msg.content.strip())    
-
-    return JSONResponse({"answers": answers}, status_code=200)
+        return JSONResponse({"answers": answers}, status_code=200)
 
 @app.get("/", include_in_schema=False)
 def root():