Commit : Updated rag.html and the routes.py for the /rag route and the requirements txt which has ctransformers and faiss-cpu

App/routes.py

@@ -14,6 +14,7 @@ from typing import List, Dict, Optional
 import time
 from datetime import datetime
 import hashlib
+import PyPDF2
 
 # Import existing components
 from App.engine.ranker import CPIRanker
@@ -427,46 +428,128 @@ def scholar_discovery():
 
 
 # ====== UPDATED RAG ROUTE (BROWSER-FREE) ======
+import os
+from flask import request, render_template
+import PyPDF2
+
+from langchain_community.llms import CTransformers
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+# Core LCEL imports
+from langchain_core.runnables import RunnablePassthrough
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+
+# ----------------------------
+# INITIALIZE EMBEDDINGS & LLM
+# ----------------------------
+embeddings = HuggingFaceEmbeddings(
+    model_name="sentence-transformers/all-MiniLM-L6-v2"
+)
+
+# Use a local GGUF model that supports text-generation
+llm = CTransformers(
+    model="TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF",
+    model_file="tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",
+    model_type="llama",
+    config={
+        'max_new_tokens': 512,
+        'temperature': 0.7,
+        'context_length': 2048,
+        'gpu_layers': 0
+    }
+)
+
+
+# ----------------------------
+# 1. HELPERS
+# ----------------------------
+def format_docs(docs):
+    """Combines retrieved document contents into one string."""
+    return "\n\n".join(doc.page_content for doc in docs)
+
+
+def extract_answer(text):
+    """
+    TinyLlama often returns the full prompt.
+    This extracts only the text after the assistant tag.
+    """
+    marker = "<|assistant|>"
+    if marker in text:
+        return text.split(marker)[-1].strip().replace("</s>", "")
+    return text.strip()
+
+
+# ----------------------------
+# 2. PROMPT: Optimized for TinyLlama Chat
+# ----------------------------
+# Note: Spacing is critical for TinyLlama to recognize the end of the user block
+prompt = ChatPromptTemplate.from_template(
+    "<|system|>\nYou are a concise assistant. Use the provided context to answer the question. If the answer isn't in the context, say you don't know.</s>\n<|user|>\nContext: {context}\nQuestion: {input}</s>\n<|assistant|>\n")
+
 
-@main_bp.route('/rag')
+# ----------------------------
+# 3. ROUTE
+# ----------------------------
+@main_bp.route('/rag', methods=['GET', 'POST'])
 def rag_search():
-    """Enhanced RAG search using browser-free data aggregation"""
-    query = request.args.get('q', '').strip()
+    query = ""
     answer = ""
     sources = []
 
-    if query:
-        # Check for HF token
-        hf_token = os.getenv('HF_TOKEN')
-
-        if not hf_token:
-            return render_template('rag.html', query=query,
-                                   answer="HF_TOKEN not configured. Please set your Hugging Face API token in environment variables.",
-                                   sources=[])
+    if request.method == 'POST':
+        query = request.form.get('q', '').strip()
+        pdf_file = request.files.get('pdf_file')
+
+        if pdf_file and pdf_file.filename.endswith('.pdf'):
+            try:
+                # 1) Read & Split PDF
+                reader = PyPDF2.PdfReader(pdf_file)
+                full_text = "".join([p.extract_text() or "" for p in reader.pages])
+
+                # Improved splitting to keep context meaningful
+                splitter = RecursiveCharacterTextSplitter(
+                    chunk_size=500,
+                    chunk_overlap=50,
+                    separators=["\n\n", "\n", ".", " "]
+                )
+                chunks = splitter.split_text(full_text)
 
-        try:
-            # Initialize systems
-            scraper = EnhancedScholarScraper()
-            rag_system = EnhancedRAGSystem(hf_token)
+                # 2) Build Local Vector DB
+                vector_db = FAISS.from_texts(chunks, embeddings)
+                retriever = vector_db.as_retriever(search_kwargs={"k": 3})
 
-            # Fetch and index data from multiple sources
-            corpus = scraper.get_comprehensive_corpus(query)
-            doc_count = rag_system.index_corpus(corpus)
+                # 3) Build MODERN LCEL Chain with Output Cleaning
+                rag_chain = (
+                        {
+                            "context": retriever | format_docs,
+                            "input": RunnablePassthrough()
+                        }
+                        | prompt
+                        | llm
+                        | StrOutputParser()
+                        | extract_answer  # Cleans the TinyLlama prompt tags
+                )
 
-            if doc_count == 0:
-                answer = f"No academic sources found for '{query}'. Try a broader search term or different keywords."
-            else:
-                # Generate synthesis
-                result = rag_system.generate_synthesis(query, k=5)
-                answer = result['answer']
-                sources = result['sources']
+                if query:
+                    # 4) Execute Chain
+                    docs = retriever.invoke(query)
+                    sources = [doc.page_content for doc in docs]
 
-        except Exception as e:
-            print(f"RAG Error: {e}")
-            answer = f"An error occurred while processing your request: {str(e)}"
+                    # This now returns only the cleaned string
+                    answer = rag_chain.invoke(query)
+                else:
+                    answer = "PDF uploaded successfully. What is your question?"
 
-    return render_template('rag.html', query=query, answer=answer, sources=sources)
+            except Exception as e:
+                print("RAG Error:", e)
+                answer = f"System Error: {str(e)}"
+        else:
+            answer = "Please upload a valid PDF file."
 
+    return render_template("rag.html", query=query, answer=answer, sources=sources)
 
 # ====== EXISTING DISCOVERY ROUTE (PRESERVED) ======
 

Templates/rag.html

@@ -1,103 +1,155 @@
 {% extends "base.html" %}
 
 {% block content %}
-<div class="min-h-screen bg-slate-50 p-8">
-    <div class="max-w-6xl mx-auto">
-        <div class="mb-12">
-            <h1 class="text-4xl font-black text-slate-900 mb-2 uppercase italic flex items-center">
-                <i class="fas fa-microchip text-indigo-600 mr-4"></i>RAG Intelligence
-            </h1>
-            <p class="text-slate-500 font-medium mb-8">Retrieval-Augmented Generation for Scientific Synthesis</p>
-
-            <form action="/rag" method="GET" class="relative group">
-                <input type="text" name="q" value="{{ query }}"
-                       placeholder="Enter research topic for synthesis (e.g., 'T cell exhaustion markers')..."
-                       class="w-full bg-white border-2 border-slate-200 rounded-2xl px-8 py-6 text-lg shadow-sm focus:border-indigo-500 focus:ring-4 focus:ring-indigo-50/50 transition-all outline-none pr-48">
-                <button type="submit" class="absolute right-4 top-4 bottom-4 bg-indigo-600 text-white px-10 rounded-xl font-bold hover:bg-indigo-700 hover:shadow-lg transition-all active:scale-95">
-                    Synthesize
-                </button>
-            </form>
-
-            <div class="mt-4 flex items-center gap-6 text-[10px] font-bold text-slate-400 uppercase tracking-widest">
-                <div class="flex items-center gap-2">
-                    <span class="w-2 h-2 rounded-full bg-green-500 animate-pulse"></span>
-                    Hugging Face Inference Active
-                </div>
-                <div class="flex items-center gap-2">
-                    <i class="fas fa-database text-indigo-400"></i>
-                    In-Memory Vector Store
+<div class="min-h-screen bg-[#f8fafc] p-8 font-sans">
+    <div class="max-w-7xl mx-auto">
+        <div class="mb-10 flex items-center justify-between">
+            <div class="flex items-center gap-3">
+                <div class="bg-blue-600 p-2 rounded-lg">
+                    <i class="fas fa-microscope text-white text-xl"></i>
                 </div>
+                <h1 class="text-2xl font-bold text-[#1e293b] tracking-tight">Qsearch Intelligence</h1>
+            </div>
+            <div class="flex gap-6 text-sm font-medium text-slate-500">
+                <span class="hover:text-blue-600 cursor-pointer flex items-center gap-2"><i class="fas fa-database"></i> Artifacts</span>
+                <span class="hover:text-blue-600 cursor-pointer flex items-center gap-2"><i class="fas fa-file-alt"></i> Publications</span>
+                <span class="hover:text-blue-600 cursor-pointer flex items-center gap-2"><i class="fas fa-graduation-cap"></i> Scholar</span>
+                <button class="bg-slate-100 px-4 py-2 rounded-md text-slate-700 hover:bg-slate-200 transition-all">Analysis Tools <i class="fas fa-chevron-down ml-1 text-xs"></i></button>
             </div>
         </div>
 
-        {% if answer %}
-        <div class="grid grid-cols-1 lg:grid-cols-12 gap-10 items-start">
-            <div class="lg:col-span-8 space-y-6">
-                <div class="bg-white p-10 rounded-[2.5rem] border border-slate-200 shadow-sm relative overflow-hidden">
-                    <div class="absolute top-0 right-0 w-32 h-32 bg-indigo-50/50 rounded-bl-full -mr-16 -mt-16"></div>
+        <div class="bg-white rounded-3xl shadow-sm border border-slate-200 p-2 mb-10">
+            <form action="/rag" method="POST" enctype="multipart/form-data" onsubmit="return showLoading()" class="flex items-center gap-2">
+                <label class="flex items-center justify-center px-6 py-4 border-r border-slate-100 cursor-pointer hover:bg-slate-50 transition-colors">
+                    <i class="fas fa-paperclip text-slate-400 mr-2"></i>
+                    <span id="file-name" class="text-sm font-semibold text-slate-500 truncate max-w-[120px]">Upload PDF</span>
+                    <input type="file" name="pdf_file" id="pdf_file" accept=".pdf" class="hidden" onchange="updateFileName(this)" required />
+                </label>
 
-                    <div class="flex items-center justify-between mb-8 relative">
-                        <h3 class="text-xs font-black text-indigo-600 uppercase tracking-[0.2em]">Research Synthesis</h3>
-                        <span class="text-[10px] font-bold text-slate-400 bg-slate-100 px-3 py-1 rounded-full">Llama-3-8B</span>
-                    </div>
+                <input type="text" name="q" id="query-input" placeholder="Ask anything about your document..."
+                       class="flex-1 px-6 py-4 text-slate-700 outline-none placeholder:text-slate-400" required>
 
-                    <div class="prose prose-indigo max-w-none text-slate-700 text-lg leading-relaxed">
-                        {{ answer | replace('\n', '<br>') | safe }}
+                <button type="submit" id="submit-btn" class="bg-blue-600 text-white px-8 py-4 rounded-2xl font-bold hover:bg-blue-700 transition-all flex items-center gap-3 mr-2">
+                    <span id="btn-text">Analyze</span>
+                    <div id="loading-spinner" class="hidden animate-spin h-4 w-4 border-2 border-white border-t-transparent rounded-full"></div>
+                </button>
+            </form>
+        </div>
+
+        {% if answer %}
+        <div id="results-area" class="grid grid-cols-1 lg:grid-cols-12 gap-8 animate-fade-in">
+            <div class="lg:col-span-8">
+                <div class="bg-white p-10 rounded-[2rem] border border-slate-200 shadow-sm min-h-[400px]">
+                    <div class="flex items-center justify-between mb-8">
+                        <h3 class="text-[11px] font-black text-indigo-500 uppercase tracking-[0.2em]">Generated Answer</h3>
+                        <button onclick="copyAnswer()" class="flex items-center gap-2 text-xs font-bold text-slate-400 hover:text-slate-600 bg-slate-50 px-3 py-1.5 rounded-md transition-all">
+                            <i class="fas fa-copy"></i> Copy
+                        </button>
                     </div>
 
-                    <div class="mt-10 pt-8 border-t border-slate-100 flex items-center justify-between">
-                        <p class="text-[11px] text-slate-400 font-medium">Verified against top {{ sources|length }} peer-reviewed profiles</p>
-                        <button class="text-indigo-600 font-bold text-xs hover:underline">Export Analysis <i class="fas fa-download ml-1"></i></button>
+                    <div id="answer-content" class="text-[#334155] text-lg leading-relaxed prose prose-slate max-w-none">
+                        {{ answer | safe }}
                     </div>
                 </div>
             </div>
 
-            <div class="lg:col-span-4 space-y-4">
-                <div class="flex items-center justify-between mb-2">
-                    <h3 class="text-xs font-black text-slate-400 uppercase tracking-widest">Primary Evidence</h3>
-                    <i class="fas fa-filter text-slate-300 text-xs"></i>
+            <div class="lg:col-span-4 space-y-6">
+                <div class="flex items-center justify-between px-2">
+                    <h3 class="text-[11px] font-black text-slate-400 uppercase tracking-[0.2em]">Supporting Context</h3>
+                    {% if sources %}
+                    <span class="text-[10px] bg-indigo-50 text-indigo-600 px-2.5 py-1 rounded-full font-bold">
+                        {{ sources|length }} snippets
+                    </span>
+                    {% endif %}
                 </div>
 
-                {% for source in sources %}
-                <div class="bg-white p-6 rounded-2xl border border-slate-200 shadow-sm group hover:border-indigo-500 hover:shadow-md transition-all relative overflow-hidden">
-                    <div class="absolute top-0 right-0 bg-slate-900 text-white px-3 py-1 text-[9px] font-black italic rounded-bl-xl group-hover:bg-indigo-600 transition-colors">
-                        #{{ loop.index }}
-                    </div>
-
-                    <h4 class="font-bold text-slate-900 mb-1 group-hover:text-indigo-600 transition-colors pr-8 leading-tight">
-                        {{ source.name }}
-                    </h4>
-                    <p class="text-[11px] text-slate-500 italic mb-4 line-clamp-1">{{ source.affiliation }}</p>
-
-                    <div class="flex items-center justify-between mt-auto">
-                        <a href="{{ source.profile_url }}" target="_blank" class="text-[10px] bg-slate-50 text-slate-600 px-3 py-2 rounded-lg font-bold hover:bg-indigo-50 hover:text-indigo-600 transition-colors">
-                            Full Profile <i class="fas fa-external-link-alt ml-1 opacity-50"></i>
-                        </a>
-                        <div class="text-[10px] font-black text-indigo-400 uppercase">Specialist</div>
-                    </div>
-                </div>
-                {% endfor %}
-
-                <div class="bg-indigo-600 p-6 rounded-2xl text-white shadow-lg shadow-indigo-200">
-                    <p class="text-xs font-bold opacity-80 mb-2 uppercase tracking-widest">Methodology</p>
-                    <h5 class="text-sm font-black italic mb-4">Composite Performance Index</h5>
-                    <p class="text-[10px] leading-relaxed opacity-90">
-                        This synthesis is generated by embedding researcher profiles into a vector space and retrieving context based on topical density.
-                    </p>
-                </div>
-            </div>
-        </div>
-        {% elif query %}
-        <div class="bg-white border-2 border-dashed border-slate-200 rounded-[3rem] py-32 text-center">
-            <div class="max-w-xs mx-auto">
-                <div class="w-16 h-16 bg-slate-100 rounded-full flex items-center justify-center mx-auto mb-6 text-slate-300">
-                    <i class="fas fa-search text-2xl"></i>
+                <div class="space-y-4">
+                    {% if sources %}
+                        {% for source in sources %}
+                        <div class="bg-white p-6 rounded-2xl border border-slate-100 shadow-sm hover:shadow-md transition-all group relative">
+                            <div class="flex items-center justify-between mb-4">
+                                <span class="text-[10px] font-black text-slate-300 uppercase tracking-widest">Chunk {{ loop.index }}</span>
+                                <button onclick="copySnippet({{ loop.index - 1 }})" class="text-slate-300 hover:text-indigo-500 transition-colors">
+                                    <i class="fas fa-copy text-xs"></i>
+                                </button>
+                            </div>
+                            <p class="text-[13px] text-slate-500 leading-relaxed snippet-text italic" data-full-text="{{ source }}">
+                                "{{ source[:220] }}{% if source|length > 220 %}...{% endif %}"
+                            </p>
+                            {% if source|length > 220 %}
+                            <button onclick="toggleSnippet({{ loop.index - 1 }})" class="text-[10px] text-indigo-500 font-bold mt-3 uppercase tracking-wider hover:underline">
+                                Show More
+                            </button>
+                            {% endif %}
+                        </div>
+                        {% endfor %}
+                    {% else %}
+                        <div class="text-center py-10">
+                            <i class="fas fa-quote-right text-slate-200 text-3xl mb-3"></i>
+                            <p class="text-sm text-slate-400 italic">No snippets found.</p>
+                        </div>
+                    {% endif %}
                 </div>
-                <h3 class="text-xl font-bold text-slate-800 mb-2">Insufficient context found</h3>
-                <p class="text-slate-400 text-sm">Our crawler could not verify enough specialist profiles for "{{ query }}". Try a broader scientific term.</p>
             </div>
         </div>
         {% endif %}
     </div>
 </div>
+
+<script>
+function updateFileName(input) {
+    const fileName = document.getElementById('file-name');
+    if (input.files && input.files[0]) {
+        fileName.textContent = input.files[0].name;
+        fileName.classList.remove('text-slate-500');
+        fileName.classList.add('text-blue-600');
+    }
+}
+
+function showLoading() {
+    const btnText = document.getElementById('btn-text');
+    const spinner = document.getElementById('loading-spinner');
+    const btn = document.getElementById('submit-btn');
+
+    btn.disabled = true;
+    btn.classList.add('opacity-80');
+    btnText.textContent = 'Analyzing...';
+    spinner.classList.remove('hidden');
+    return true;
+}
+
+function copyAnswer() {
+    const text = document.getElementById('answer-content').innerText;
+    navigator.clipboard.writeText(text);
+    alert('Answer copied!');
+}
+
+function toggleSnippet(index) {
+    const snippets = document.querySelectorAll('.snippet-text');
+    const buttons = document.querySelectorAll('.snippet-text + button');
+    const snippet = snippets[index];
+    const button = buttons[index];
+    const fullText = snippet.getAttribute('data-full-text');
+
+    if (snippet.classList.contains('expanded')) {
+        snippet.textContent = `"${fullText.substring(0, 220)}..."`;
+        snippet.classList.remove('expanded');
+        button.textContent = 'Show More';
+    } else {
+        snippet.textContent = `"${fullText}"`;
+        snippet.classList.add('expanded');
+        button.textContent = 'Show Less';
+    }
+}
+</script>
+
+<style>
+@keyframes fade-in {
+    from { opacity: 0; transform: translateY(20px); }
+    to { opacity: 1; transform: translateY(0); }
+}
+.animate-fade-in { animation: fade-in 0.6s cubic-bezier(0.22, 1, 0.36, 1) forwards; }
+body { background-color: #f8fafc; }
+.prose p { margin-bottom: 1.5rem; }
+</style>
 {% endblock %}

requirements.txt

@@ -22,7 +22,11 @@ Bio
 langchain-huggingface
 langchain-core
 huggingface-hub
+PyPDF2
 
 scholarly
 feedparser
-python-dateutil
+python-dateutil
+
+ctransformers
+faiss-cpu