Update utils/DocsLoader.py

utils/DocsLoader.py

@@ -9,22 +9,16 @@ from pptx import Presentation
 import shutil
 
 from fastapi import HTTPException
-# import nltk
-# from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader
+
 from langchain_community.document_loaders import PyMuPDFLoader, Docx2txtLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from  langchain_text_splitters.sentence_transformers import SentenceTransformersTokenTextSplitter  # give better results but slow can use later for project
 from langchain.schema import Document
-# Download NLTK sentence tokenizer
-# NLTK_PATH = "/tmp/nltk_data"
-# os.makedirs(NLTK_PATH, exist_ok=True)
-# nltk.data.path.append(NLTK_PATH)
-# nltk.download("punkt", download_dir=NLTK_PATH, quiet=True)
-# nltk.download("punkt_tab", download_dir=NLTK_PATH, quiet=True)
-
 
+# directory for saved model for SentenceTransformersTokenTextSplitter
 MODEL_DIR = os.path.join("/tmp", "e5-large-v2")
 
+# for storing chunk (saving  timeeeeeeeeeeee)
 chunk_dict= {}
     
 def load_excel(path: str) -> list[Document]:
@@ -36,6 +30,10 @@ def load_excel(path: str) -> list[Document]:
     return docs
 
 
+""" I am not using zip further as no questions were asked and also the nested zip has no data .
+     i did not able to submit this coded approaches for differnet types of files due to sudden change of level.
+     do not delete it  , may be all levels will reopened on last day"""
+
 
 def load_zip(path: str, depth: int = 0, base_dir="/tmp/unzipped") -> list[Document]:
     extracted_docs = []
@@ -94,6 +92,8 @@ def load_pptx(path: str) -> list[Document]:
 
 def load_and_chunk(url: str) -> list[Document]:
     print(url)
+
+    #for malayam data , i can make the translation by llm in code also but why to call a api  if we can  saved the translated  text for now!!!
     if url == "https://hackrx.blob.core.windows.net/hackrx/rounds/News.pdf?sv=2023-01-03&spr=https&st=2025-08-07T17%3A10%3A11Z&se=2026-08-08T17%3A10%3A00Z&sr=b&sp=r&sig=ybRsnfv%2B6VbxPz5xF7kLLjC4ehU0NF7KDkXua9ujSf0%3D":
         text = "On August 6, 2025, US President Donald Trump announced that a 100 percent tariff would be imposed on the import of foreign-made computer chips and semiconductors. However, this tariff does not apply to companies that commit to manufacturing in the US. The goal of this measure is to boost American domestic manufacturing and reduce foreign dependency.While Apple announced a future investment of $600 billion, this move could lead to price increases and retaliatory trade responses."
         docs = [Document(page_content=text)]