Update RAG backend and gitignore

.gitignore

@@ -0,0 +1,68 @@
+
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.egg-info/
+.eggs/
+
+
+# Virtual Environments
+.venv/
+venv/
+env/
+ENV/
+
+
+# Environment Variables
+.env
+.env.*
+*.key
+
+
+# Jupyter
+.ipynb_checkpoints/
+
+
+# Logs
+*.log
+
+
+# OS Files
+.DS_Store
+Thumbs.db
+
+
+# IDE / Editor
+.vscode/
+.idea/
+*.swp
+
+
+# Uploaded / User Files
+upload/
+uploads/
+*.pdf
+
+
+# Vector Stores (FAISS)
+faiss_index/
+vectorstore/
+*.faiss
+*.pkl
+
+
+# Streamlit
+.streamlit/secrets.toml
+
+
+# Cache / Temp
+.cache/
+tmp/
+temp/
+
+
+# Build / Distribution
+build/
+dist/

backend/api.py

@@ -21,8 +21,6 @@ def get_vectorstore():
     return load_documents(embedding_model=get_embeddings())
 
 
-
-
 BASE_DIR = Path("/app")
 upload_dir = BASE_DIR / "uploads"
 upload_dir.mkdir(parents=True, exist_ok=True)
@@ -50,12 +48,6 @@ system_stats = {
     "start_time": datetime.now().isoformat()
 }
 
-@app.on_event("startup")
-def startup_event():
-    print("🔄 Preloading embedding model...")
-    get_embeddings()
-    print("✅ Embedding model loaded")
-
 # Info about API
 @app.get("/")
 async def root():
@@ -112,33 +104,28 @@ async def get_stats():
 # This Endpoint upload Pdf and store into VectorDatabase
 @app.post("/upload")
 async def upload_file(file: UploadFile = File(...)):
-    try:
-        if not file.filename.endswith(".pdf"):
-            raise HTTPException(status_code=400, detail="Only PDF files are supported")
+    if not file.filename.endswith(".pdf"):
+        raise HTTPException(status_code=400, detail="Only PDF files are supported")
 
-        file_path = upload_dir / file.filename
+    file_path = upload_dir / file.filename
 
-        with open(file_path, "wb") as f:
-            shutil.copyfileobj(file.file, f)
+    with open(file_path, "wb") as f:
+        shutil.copyfileobj(file.file, f)
 
-        chunked_docs = get_chunked_docs(file_path)
+    chunked_docs = get_chunked_docs(file_path)
 
-        if not chunked_docs:
-            raise HTTPException(status_code=500, detail="No content extracted from PDF")
+    if not chunked_docs:
+        raise HTTPException(status_code=500, detail="No content extracted from PDF")
 
-        store_documents(chunked_docs, get_embeddings())
-
-        system_stats["total_uploads"] += 1
-
-        return {
-            "message": "PDF uploaded and indexed successfully",
-            "chunks_created": len(chunked_docs)
-        }
-
-    except Exception as e:
-        print("❌ UPLOAD ERROR:", str(e))  # <-- shows in HF logs
-        raise HTTPException(status_code=500, detail=str(e))
+    store_documents(chunked_docs, get_embeddings())
+    
+    # INCREMENT THE COUNTER HERE!
+    system_stats["total_uploads"] += 1
 
+    return {
+        "message": "PDF uploaded and indexed successfully",
+        "chunks_created": len(chunked_docs)
+    }
     
 from pydantic import BaseModel
 

rag/chain.py

@@ -50,6 +50,7 @@ prompt = ChatPromptTemplate.from_template(
 Answer the question strictly using the information contained in the document excerpts below.
 Do not mention the phrases "provided context", "given context", or similar meta-references.
 Do not include conversational language or assumptions.
+Do not use any HTML tags or formatting in your response.
 
 Writing guidelines:
 - Use a formal, neutral, and analytical tone.
@@ -57,12 +58,6 @@ Writing guidelines:
 - If information is missing, clearly state that it is not available in the document.
 - Do not speculate or add external knowledge.
 
-Citation rules:
-- List citations in a separate section highlighted with blue.
-- Each citation must include page number and table/figure/image reference if available.
-- Use this format exactly:
-  • Page X, Table/Figure/Image Y (if applicable)
-
 <Document Excerpts>
 {context}
 </Document Excerpts>
@@ -71,7 +66,6 @@ Question:
 {input}
 """
 )
-
 # Get Retrieval chain
 def get_rag_chain(retriever):
     chain = (

rag/combine.py

@@ -4,13 +4,20 @@ import camelot
 import pytesseract
 from PIL import Image
 import io
+import re
+
+
+def clean_text(text: str) -> str:
+    text = re.sub(r'<[^>]+>', '', text)      # strip <font>, <b>, <i>, etc.
+    text = re.sub(r'[ \t]+', ' ', text)      # normalize spaces/tabs
+    text = re.sub(r'\n{3,}', '\n\n', text)   # collapse excessive newlines
+    return text.strip()
 
 
 # Raw Documents
 def raw_document_text(pdf_path: str):
     documents = []
 
-    # Open PDF
     with pdfplumber.open(pdf_path) as pdf:
         doc_fitz = fitz.open(pdf_path)
 
@@ -20,14 +27,13 @@ def raw_document_text(pdf_path: str):
             text = page.extract_text()
             if text:
                 documents.append({
-                    "content": text,
+                    "content": clean_text(text),
                     "metadata": {
                         "page": page_index,
                         "type": "text"
                     }
                 })
 
-            
             # TABLES
             tables = camelot.read_pdf(
                 pdf_path,
@@ -38,7 +44,7 @@ def raw_document_text(pdf_path: str):
             for t_idx, table in enumerate(tables):
                 table_text = table.df.to_string(index=False)
                 documents.append({
-                    "content": table_text,
+                    "content": clean_text(table_text),
                     "metadata": {
                         "page": page_index,
                         "type": "table",
@@ -46,7 +52,6 @@ def raw_document_text(pdf_path: str):
                     }
                 })
 
-            
             # IMAGES + OCR
             page_fitz = doc_fitz[page_index - 1]
             images = page_fitz.get_images(full=True)
@@ -61,7 +66,7 @@ def raw_document_text(pdf_path: str):
 
                 if ocr_text.strip():
                     documents.append({
-                        "content": ocr_text,
+                        "content": clean_text(ocr_text),
                         "metadata": {
                             "page": page_index,
                             "type": "image",
@@ -69,6 +74,4 @@ def raw_document_text(pdf_path: str):
                         }
                     })
 
-    return documents
-
-
+    return documents

rag/smart_search.py

@@ -0,0 +1,107 @@
+from collections import defaultdict
+
+from langchain_core.documents import Document
+
+
+# Raw Documents to Langchain Documents
+def get_langchain_docs(docs:str):
+    lc_docs = []
+    for doc in docs:
+        document = Document(
+            page_content=doc['content'],
+            metadata=doc['metadata']
+        )
+        lc_docs.append(document)
+    return lc_docs
+
+
+
+docs = {
+    1: "Artificial intelligence and machine learning are transforming modern software systems by enabling automated decision making and intelligent data analysis.",
+    2: "Smart parking systems use sensors, real time data processing, and predictive analytics to efficiently allocate parking slots and reduce traffic congestion in urban cities.",
+    3: "Data science combines statistics, programming, and domain knowledge to extract meaningful insights from large datasets for business and research applications."
+}
+
+
+index = defaultdict(set)
+for doc_id, text in docs.items():
+    words = text.lower().split()
+    for word in words:
+        index[word].add(doc_id)
+
+
+class TrieNode:
+    def __init__(self):
+        self.children = {}
+        self.is_end = False
+
+class Trie:
+    def __init__(self):
+        self.root = TrieNode()
+
+    def insert(self, word):
+        node = self.root
+        for ch in word:
+            if ch not in node.children:
+                node.children[ch] = TrieNode()
+            node = node.children[ch]
+        node.is_end = True
+
+def autocomplete(prefix):
+    results = []
+    trie = Trie()
+    for text in docs.values():
+        for word in text.split():
+             trie.insert(word)
+
+    def dfs(node, path):
+        if node.is_end:
+            results.append(path)
+        for ch, nxt in node.children.items():
+            dfs(nxt, path + ch)
+
+    node = trie.root
+    for ch in prefix:
+        if ch not in node.children:
+            return []
+        node = node.children[ch]
+
+    dfs(node, prefix)
+    return results
+
+def auto_complete(query):
+    result = []
+
+    for q in query.split():
+        try:
+            result.append(autocomplete(q)[0])
+        except IndexError:
+            continue
+
+    return " ".join(result)
+
+def ranked_search(query):
+    words = query.lower().split()
+    score = {}
+
+    for w in words:
+        for doc in index[w]:
+            score[doc] = score.get(doc, 0) + 1
+
+    return sorted(score.items(), key=lambda x: x[1], reverse=True)
+
+def get_doc(query):
+    ranked = ranked_search(query)
+    docs = []
+    if len(ranked)>0:
+        for i in range(len(ranked)):
+            docs.append(ranked[i][0])
+    return docs[:2]
+  
+query = "Smar par system combines  Data science statistics"  
+def final_docs_Search(query):
+    sent = auto_complete(query)
+    result = get_doc(sent)
+    return result
+
+print(final_docs_Search(query))