Update utils/DocsLoader.py

utils/DocsLoader.py

@@ -7,6 +7,7 @@ from PIL import Image
 import pytesseract
 from pptx import Presentation
 import shutil
+from langdetect import detect
 
 from fastapi import HTTPException
 
@@ -20,6 +21,12 @@ MODEL_DIR = os.path.join("/tmp", "e5-large-v2")
 
 # for storing chunk (saving  timeeeeeeeeeeee)
 chunk_dict= {}
+
+GOOGLE_API_KEY2 = os.getenv("gemini_api_key2")
+llm = ChatGoogleGenerativeAI(
+                               model="gemini-1.5-flash",
+                               api_key=GOOGLE_API_KEY2,
+                           )
     
 def load_excel(path: str) -> list[Document]:
     dfs = pd.read_excel(path, sheet_name=None)
@@ -92,16 +99,7 @@ def load_pptx(path: str) -> list[Document]:
 
 def load_and_chunk(url: str) -> list[Document]:
     print(url)
-
-    #for malayam data , i can make the translation by llm in code also but why to call a api  if we can  saved the translated  text for now!!!
-    if url == "https://hackrx.blob.core.windows.net/hackrx/rounds/News.pdf?sv=2023-01-03&spr=https&st=2025-08-07T17%3A10%3A11Z&se=2026-08-08T17%3A10%3A00Z&sr=b&sp=r&sig=ybRsnfv%2B6VbxPz5xF7kLLjC4ehU0NF7KDkXua9ujSf0%3D":
-        text = "On  August  6,  2025,  US  President  Donald  Trump  announced  that  he  would  impose  a  100  percent  tariff  on  imports  of  computer  chips  and  semiconductors  made  abroad.  However,  the  tariff  would  not  apply  to  companies  that  have  committed  to  manufacturing  in  the  US.  The  aim  of  this  measure  is  to  boost  domestic  manufacturing  in  the  US  and  reduce  foreign  dependence.  While  Apple  announced  a $600  billion  investment,  the  continuation  of  this  windfall  opens  the  door  to  price  increases  and  anti-trade  reactions."
-        docs = [Document(page_content=text)]
-        full_text = "\n".join([doc.page_content for doc in docs])
-        splitter = SentenceTransformersTokenTextSplitter(
-            model_name=MODEL_DIR, tokens_per_chunk=512, chunk_overlap=90
-        )
-        chunk_dict[url] = splitter.create_documents([full_text])        
+       
     if url not in chunk_dict:
         print("processing new url")
         resp = requests.get(url)
@@ -162,13 +160,30 @@ def load_and_chunk(url: str) -> list[Document]:
             if 'tmp_path' in locals() and os.path.exists(tmp_path):
                 os.remove(tmp_path)
 
-        full_text = "\n".join([doc.page_content for doc in docs])
+        
+        
+        min_len = 100 if len(docs[0].page_content) >= 100 else len(docs[0].page_content)
+        lang = detect(docs[0].page_content[:min_len])
+
+        text = ""
+        if lang != "en":
+            for doc in docs:
+                translated = llm.invoke(f"Translate this into English:\n{doc.page_content}")
+                text = text + translated.content
+
+        else:
+            full_text = "\n".join([doc.page_content for doc in docs])
+            
         splitter = SentenceTransformersTokenTextSplitter(
             model_name=MODEL_DIR, tokens_per_chunk=512, chunk_overlap=90
         )
-        chunk_dict[url] = splitter.create_documents([full_text])
+        
+        if lang !="en":
+            chunk_dict[url] = splitter.create_documents([text])
+        else:    
+            chunk_dict[url] = splitter.create_documents([full_text])
+            
         return chunk_dict[url]
     else:
         print("stored chunk")
-        return chunk_dict[url]
-
+        return chunk_dict[url]