Add LangChain RAG pipeline for custom PDF advisory with LFS tracking

.gitattributes

@@ -1,2 +1,3 @@
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.png filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text

1_agro.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe196417aa478f1807235c513e6e9c4f23a3c52ecc7fcc4dc42703b7de53b2a0
+size 8617145

2-agro.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e674cd3b6a214696d24d6876892cee6ffab3ae7e9499017f573c25690e1b901e
+size 17831258

3-agrogpt.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adbb66a8efc7bc24cb6b4afc99782519cf41d8b1ca0414c486db6627d7bc5c7f
+size 4316300

4-agrogpt.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e5c45b5cefa9eb34f7abba0e72e91ea20feba58e67bfbd0f0706f980d0df48d
+size 6330107

app.py

@@ -18,6 +18,7 @@ import requests
 # Import the new model functionality
 from model import check_ollama_connection, generate_with_ollama, analyze_image_for_disease
 from weather import get_current_weather, TN_DISTRICTS
+from rag_engine import initialize_knowledge_base, query_rag
 
 app = Flask(__name__)
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0  # Disable static file caching
@@ -114,6 +115,10 @@ def check_backend_status():
         status_message = "Error: Could not connect to Ollama. Ensure GROQ_API_KEY is set in .env."
         print(status_message, flush=True)
 
+    # Initialize the vector DB in the background
+    print("Triggering background RAG knowledge base initialization...", flush=True)
+    threading.Thread(target=initialize_knowledge_base).start()
+
 @app.route('/')
 def index():
     """Serve the main page"""
@@ -219,15 +224,21 @@ def ask_question():
     try:
         # Construct prompt
         weather_section = f"\n\n[Weather Context]: {weather_context.strip()}" if weather_context.strip() else ""
+        
+        # Fetch RAG context
+        print("Fetching expert PDF knowledge context...", flush=True)
+        rag_context = query_rag(question)
+        rag_section = f"\n\n[Expert PDF Advisory Knowledge Base Context]:\n{rag_context}\n\nUse this information above to enhance your answer if relevant." if rag_context else ""
+
         full_prompt = (
-            "You are AgroGPT, an expert agriculture assistant. "
+            "You are AgroGPT, an expert agriculture assistant with access to detailed agricultural advisory documents. "
             "Answer the following question clearly and concisely in plain text. "
             "Structure your response EXACTLY as follows:\n"
-            "1. Provide a helpful answer in English.\n"
+            "1. Provide a helpful, robust answer in English. Ensure you include relevant information from the knowledge base if available.\n"
             "2. Then write the header 'Malayalam Summary:' followed by the FULL answer translated into native Malayalam script (മലയാളം). Do NOT use English/Latin letters for Malayalam.\n"
             "3. Then write the header 'Tamil Summary:' followed by the FULL answer translated into native Tamil script (தமிழ்). Do NOT use English/Latin letters for Tamil.\n"
             "Use double line breaks between each section. Do NOT use markdown, asterisks, or bullet points.\n\n"
-            f"Question: {question}{weather_section}\n\nAnswer:"
+            f"Question: {question}{weather_section}{rag_section}\n\nAnswer:"
         )
         
         print(f"Asking Ollama: {question}", flush=True)

rag_engine.py

@@ -0,0 +1,103 @@
+import os
+from pathlib import Path
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+
+# Path to local PDFs
+PDF_FILES = [
+    "1_agro.pdf",
+    "2-agro.pdf",
+    "3-agrogpt.pdf",
+    "4-agrogpt.pdf"
+]
+
+# Writeable path on Hugging Face for the vector store
+VECTORSTORE_DIR = "/tmp/vectorstore"
+
+# Global variable to hold the FAISS index in memory
+_vector_store = None
+
+def get_embeddings_model():
+    """Return the HuggingFace embeddings model. Uses a lightweight fast model."""
+    # Using all-MiniLM-L6-v2 as it's very fast and effective for semantic search
+    return HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+
+def initialize_knowledge_base():
+    """
+    Checks if the vector database exists in /tmp/vectorstore.
+    If not, it reads the PDFs, chunks them, generates embeddings, and saves the DB.
+    """
+    global _vector_store
+    
+    if os.path.exists(VECTORSTORE_DIR) and os.path.exists(os.path.join(VECTORSTORE_DIR, "index.faiss")):
+        print(f"Loading existing vector database from {VECTORSTORE_DIR}...")
+        _vector_store = FAISS.load_local(VECTORSTORE_DIR, get_embeddings_model(), allow_dangerous_deserialization=True)
+        return _vector_store
+        
+    print("Vector database not found. Initializing knowledge base (this may take a minute)...", flush=True)
+    os.makedirs(VECTORSTORE_DIR, exist_ok=True)
+    
+    documents = []
+    for pdf_file in PDF_FILES:
+        try:
+            if os.path.exists(pdf_file):
+                print(f"Parsing {pdf_file}...", flush=True)
+                loader = PyPDFLoader(pdf_file)
+                documents.extend(loader.load())
+            else:
+                print(f"Warning: {pdf_file} not found in the root directory.", flush=True)
+        except Exception as e:
+            print(f"Error parsing {pdf_file}: {e}", flush=True)
+
+    if not documents:
+        print("No documents were loaded. Vector database initialization skipped.", flush=True)
+        return None
+
+    print(f"Total pages loaded: {len(documents)}. Splitting text...", flush=True)
+    
+    # Split the documents into manageable chunks
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000,
+        chunk_overlap=200,
+        length_function=len
+    )
+    chunks = text_splitter.split_documents(documents)
+    
+    print(f"Created {len(chunks)} text chunks. Generating embeddings...", flush=True)
+    
+    # Generate embeddings and build FAISS index
+    embeddings = get_embeddings_model()
+    _vector_store = FAISS.from_documents(chunks, embeddings)
+    
+    # Save for subsequent requests
+    _vector_store.save_local(VECTORSTORE_DIR)
+    print(f"Vector database built and saved to {VECTORSTORE_DIR} successfully.", flush=True)
+    
+    return _vector_store
+
+def query_rag(question: str, k: int = 3) -> str:
+    """
+    Searches the FAISS vector database for chunks related to the question.
+    Returns a formatted string containing the retrieved context.
+    """
+    global _vector_store
+    
+    if not _vector_store:
+        _vector_store = initialize_knowledge_base()
+        
+    if not _vector_store:
+        # If it's still None (e.g. PDFs missing or error), return empty context
+        return ""
+        
+    try:
+        # Perform similarity search
+        docs = _vector_store.similarity_search(question, k=k)
+        
+        # Format the retrieved documents into a single context string
+        context = "\n\n".join([f"[Source: {doc.metadata.get('source', 'Unknown')}]\n{doc.page_content}" for doc in docs])
+        return context
+    except Exception as e:
+        print(f"Error querying RAG: {e}", flush=True)
+        return ""

requirements.txt

@@ -16,6 +16,27 @@ torchvision>=0.15.0
 Pillow>=9.0.0
 numpy>=1.21.0
 
+# Data processing
+pandas>=1.3.0
+scikit-learn>=1.0.0
+# AgroGPT Requirements
+# Core web framework
+Flask==3.1.2
+Werkzeug==3.1.3
+
+# LLM Interface
+# ollama>=0.1.0
+groq>=0.9.0
+
+# Machine Learning and AI (for Vision Model)
+# Restored for Azure deployment
+torch>=2.0.0
+torchvision>=0.15.0
+
+# Image processing
+Pillow>=9.0.0
+numpy>=1.21.0
+
 # Data processing
 pandas>=1.3.0
 scikit-learn>=1.0.0
@@ -36,3 +57,12 @@ Flask-Login>=0.6.3
 Flask-SQLAlchemy>=3.1.1
 Flask-Limiter>=3.7.0
 psycopg2-binary>=2.9.9
+
+# RAG and LangChain requirements
+langchain>=0.1.0
+langchain-community>=0.0.10
+langchain-groq>=0.0.1
+langchain-huggingface>=0.0.1
+pypdf>=4.0.0
+faiss-cpu>=1.7.0
+sentence-transformers>=2.2.0