removing gguf file

.flaskenv

@@ -1,2 +0,0 @@
-FLASK_APP=app.py
-FLASK_ENV=development

.gitignore

@@ -23,6 +23,9 @@ parts/
 sdist/
 var/
 wheels/
+raq/
+uploads/
+chroma_temp/
 share/python-wheels/
 *.egg-info/
 .installed.cfg

Custom_RAG

@@ -0,0 +1 @@
+Subproject commit b93cb3e02cd451d140ce30f823dc68db13d584c4

Dockerfile

@@ -1,14 +1,16 @@
-FROM python:3.11
-
-RUN useradd -m -u 1000 user
-USER user
-ENV PATH="/home/user/.local/bin:$PATH"
+From Python 3.12.7
 
+# Step 2: Set working directory in the container
 WORKDIR /app
 
-COPY --chown=user ./requirements.txt requirements.txt
-RUN pip install --no-cache-dir --upgrade -r requirements.txt
+# Step 3: Copy your app files into the container
+COPY . /app
+
+# Step 4: Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
 
-COPY --chown=user . /app
+# Step 5: Expose a port (optional, but useful)
 EXPOSE 7860
-CMD ["gunicorn", "-b", "0.0.0.0:7860", "app:app"]
+
+# Step 6: Command to run your app
+CMD ["python", "app.py"]

app.py

@@ -34,7 +34,9 @@ def upload_pdf():
 @app.route ("/stream", methods=["POST"])
 @cross_origin()
 def stream_answer():
+    
     question = request.json.get("question", "")
+    print (question)
     if not question.strip():
         return jsonify({"error": "Empty question"}), 400
 
@@ -57,6 +59,18 @@ def ask():
         return jsonify({"error": str(e)}),500
     return jsonify({"message": answer})
 
+@app.route("/stream_answer",methods=["POST"])
+@cross_origin()
+def stream_question():
+    data = request.get_json()
+    question = data.get("question","")
+    if not question:
+        return jsonify({"error": "No question provided"}),400
+    def event_stream():
+        for token in rag.stream_answer(question=question):
+            yield token
+    return Response(event_stream(), content_type ="text/event-stream")
+
 
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=6000)
+      app.run(host="0.0.0.0", port=7860)

rag_engine.py

@@ -1,17 +1,18 @@
 import os 
 import shutil
 import tempfile
+from threading import Thread
 from langchain_community.document_loaders import PyPDFLoader
 from langchain_text_splitters import RecursiveCharacterTextSplitter
-from langchain_ollama import OllamaEmbeddings
-from langchain_community.vectorstores import Chroma 
+from langchain_community.vectorstores import Chroma
+from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain.chains import RetrievalQA
-from langchain_community.llms import Ollama
+from langchain_community.llms import HuggingFacePipeline
+from transformers import TextIteratorStreamer, AutoTokenizer, AutoModelForCausalLM, pipeline
+
 from book_title_extractor import BookTitleExtractor
 from duplicate_detector import DuplicateDetector
-from langchain_core.callbacks.base import BaseCallbackHandler
-from langchain_community.chat_models import ChatOllama
-class StreamingHanlder(BaseCallbackHandler):
+class StreamingHanlder():
     def __init__(self):
         self.buffer =[]
         self.token_callback = None
@@ -22,33 +23,34 @@ class StreamingHanlder(BaseCallbackHandler):
 
 
 class RagEngine:
-    def __init__(self, embed_model= "nomic-embed-text",llm_model="qwen:1.8b", temp_dir ="chroma_temp"):
-        self.embed_model = embed_model
-        self.llm_model = llm_model
-        self.embedding = OllamaEmbeddings(model=self.embed_model)
-        self.vectorstore = None
-        self.qa_chain = None
-        self.handler = StreamingHanlder()
-        self.llm = ChatOllama (model=self.llm_model, streaming= True, callbacks=[self.handler] )
-        
+    def _load_vectorstore(self):
+        if os.path.exists(os.path.join(self.persist_dir, "chroma.sqlite3")):
+           self.vectorstore = Chroma(
+                persist_directory=self.persist_dir,
+                embedding_function=self.embedding
+            )
+           self.retriever = self.vectorstore.as_retriever()
+    def __init__(self, persist_dir="chroma_store",embed_model= "nomic-embed-text",llm_model="qwen:1.8b", temp_dir ="chroma_temp"):
         self.temp_dir = temp_dir
-       
         os.makedirs(self.temp_dir, exist_ok=True)
-        self.title_extractor = BookTitleExtractor(llm=self.llm)
         self.duplicate_detector = DuplicateDetector()
-        if os.path.exists(os.path.join(self.temp_dir, "chroma.sqlite3")):
-            print("🔁 Loading existing Chroma vectorstore...")
-            self.vectorstore = Chroma(
-                persist_directory=self.temp_dir,
-                embedding_function=self.embedding
-            )
-            self.qa_chain = RetrievalQA.from_chain_type(
-                llm=self.llm,
-                retriever=self.vectorstore.as_retriever(),
-                return_source_documents=True
-            )
-            print("Vectorstore and QA chain restored.")
-    
+        self.title_extractor = BookTitleExtractor()    
+        self.embedding = HuggingFaceEmbeddings(
+              model_name="sentence-transformers/all-MiniLM-L6-v2"
+        )
+        
+        self.vectorstore =None
+        self.retriever = None
+        self.persist_dir = "chroma_temp"
+        self._load_vectorstore()
+        self.model_id = "Qwen/Qwen-1_8B-Chat"
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_id, trust_remote_code = True)
+        self.model = AutoModelForCausalLM.from_pretrained(self.model_id,
+                                                        trust_remote_code = True,
+                                                        device_map ="auto",
+                                                        torch_dtype = "auto")
+        self.model.eval()    
+        
     def clear_temp(self):
         shutil.rmtree(self.temp_dir,ignore_errors=True)
         os.makedirs(self.temp_dir, exist_ok=True)
@@ -84,13 +86,64 @@ class RagEngine:
         else:
             self.vectorstore.add_documents(chunks)
       
-      
-        self.qa_chain = RetrievalQA.from_chain_type(
-            llm = self.llm,
-            retriever = self.vectorstore.as_retriever(),
-            return_source_documents = True
+        self.vectorstore.persist()
+        self.retriever = self.vectorstore.as_retriever()
+    
+    def stream_answer(self, question):
+        if not self.retriever: 
+            yield "data: ❗ Please upload and index a PDF first.\n\n"
+            return
+        docs = self.retriever.get_relevant_documents(question)
+        if not docs:
+            yield "data: ❗ No relevant documents found.\n\n"
+            return
+        sources = []
+        for doc in docs:
+            title = doc.metadata.get("source", "Unknown Title")
+            page = doc.metadata.get("page", "Unknown Page")
+            sources.append(f"{title} - Page {page}")
+        context = "\n\n".join([doc.page_content for doc in docs[:3]])
+       
+        system_prompt = "You are a helpful assistant that only replies in English."
+        user_prompt = f"Context:\n{context}\n\nQuestion: {question}"
+       
+        prompt = (
+            "<|im_start|>system\nYou are a helpful assistant that only replies in English.<|im_end|>\n"
+            f"<|im_start|>user\nContext:\n{context}\n\nQuestion: {question}<|im_end|>\n"
+            "<|im_start|>assistant\n"
         )
-        
+        print (prompt)
+        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+        print("🔢 Prompt token length:", inputs['input_ids'].shape[-1])     
+        streamer = TextIteratorStreamer(
+        tokenizer=self.tokenizer,
+        skip_prompt=True,
+        skip_special_tokens=True
+        )
+        generation_args = {
+        "input_ids": inputs["input_ids"],
+        "attention_mask": inputs["attention_mask"],
+        "max_new_tokens": 512,
+        "streamer": streamer,
+        "do_sample": False,
+        "temperature": 0.0,
+        "top_p": 0.95,
+        }
+        thread = Thread(target=self.model.generate, kwargs=generation_args)
+        thread.start()
+        collected_tokens = []
+        for token in streamer:
+            if token.strip():  # Filter out whitespace
+                collected_tokens.append(token)
+                
+                yield f"{token} "
+        if sources:
+           sources_text = "\n\n📚 **Sources:**\n" + "\n".join(set(sources))
+        for line in sources_text.splitlines():
+            if  line.strip():
+                yield f"{line} \n"
+
+        yield "\n\n"
     def ask_question(self, question):
         print (question)
         if not self.qa_chain :