Initial commit - Scholar RAG Engine

.gitignore

@@ -0,0 +1,5 @@
+venv/
+__pycache__/
+*.pyc
+.env
+.DS_Store

chunking.py

@@ -0,0 +1,69 @@
+import re
+
+def chunk_text(text, source, chunk_size=120):
+
+    sentences = re.split(r'(?<=[.!?])\s+', text)
+
+    chunks = []
+    current = []
+    length = 0
+
+    for s in sentences:
+
+        s = s.strip()
+
+        # remove exam noise
+        if any(x in s for x in [
+            "APRIL/MAY",
+            "CO1",
+            "Marks",
+            "Bloom",
+            "Unit",
+            "Semester"
+        ]):
+            continue
+
+        words = s.split()
+
+        if len(words) < 5:
+            continue
+
+        if length + len(words) > chunk_size:
+
+            chunks.append({
+                "source": source,
+                "text": " ".join(current)
+            })
+
+            current = []
+            length = 0
+
+        current.append(s)
+        length += len(words)
+
+    if current:
+        chunks.append({
+            "source": source,
+            "text": " ".join(current)
+        })
+
+    return chunks
+def compress_context(text, question):
+
+    sentences = text.split(". ")
+
+    keywords = question.lower().split()
+
+    scored = []
+
+    for s in sentences:
+
+        score = sum(1 for k in keywords if k in s.lower())
+
+        scored.append((score, s))
+
+    scored.sort(reverse=True)
+
+    top = [s for _, s in scored[:3]]
+
+    return ". ".join(top)

ingestion.py

@@ -0,0 +1,24 @@
+from pypdf import PdfReader
+import re
+
+def extract_pdf(file):
+
+    reader = PdfReader(file)
+
+    text = ""
+
+    for page in reader.pages:
+
+        page_text = page.extract_text() or ""
+
+        # remove extra whitespace
+        page_text = re.sub(r"\s+", " ", page_text)
+
+        # remove exam formatting noise
+        page_text = re.sub(r"CO\d+", "", page_text)
+        page_text = re.sub(r"K\d+", "", page_text)
+        page_text = re.sub(r"\d+ Marks", "", page_text)
+
+        text += page_text + "\n"
+
+    return text

llm.py

@@ -0,0 +1,57 @@
+import requests
+import os
+
+GEMINI_API_KEY = os.getenv("GOOGLE_API_KEY")
+
+def generate_answer(context, question):
+
+    prompt = f"""
+You are answering exam questions.
+
+Use the information in the context to answer the question directly.
+
+Do NOT describe the context.
+Do NOT say "the context says".
+
+Give the final answer.
+
+Context:
+{context}
+
+Question:
+{question}
+
+Answer:
+"""
+
+    url = f"https://generativelanguage.googleapis.com/v1/models/gemini-2.5-flash:generateContent?key={GEMINI_API_KEY}"
+
+    headers = {
+        "Content-Type": "application/json"
+    }
+
+    data = {
+        "contents":[
+            {
+                "parts":[
+                    {"text": prompt}
+                ]
+            }
+        ],
+        "generationConfig":{
+            "temperature":0.3,
+            "maxOutputTokens":300
+        }
+    }
+
+    response = requests.post(url, headers=headers, json=data)
+
+    print("Gemini status:", response.status_code)
+
+    if response.status_code != 200:
+        print(response.text)
+        raise Exception("LLM failed")
+
+    result = response.json()
+
+    return result["candidates"][0]["content"]["parts"][0]["text"]

main.py

@@ -0,0 +1,90 @@
+from fastapi import FastAPI, UploadFile, Form, Request
+from fastapi.responses import HTMLResponse
+from fastapi.templating import Jinja2Templates
+
+from ingestion import extract_pdf
+from chunking import chunk_text, compress_context
+from retrieval_colbert import ColBERTRetriever
+from reranker import rerank
+from llm import generate_answer
+from scraper import scrape_url
+
+app = FastAPI()
+
+templates = Jinja2Templates(directory="templates")
+
+retriever = ColBERTRetriever()
+
+
+@app.get("/", response_class=HTMLResponse)
+async def home(request: Request):
+
+    return templates.TemplateResponse(
+        "index.html",
+        {"request": request}
+    )
+
+
+@app.post("/upload")
+async def upload(file: UploadFile):
+
+    text = extract_pdf(file.file)
+
+    chunks = chunk_text(text, file.filename)
+
+    retriever.build_index(chunks)
+
+    print("Index built with", len(chunks), "chunks")
+
+    return {"status": "indexed"}
+
+@app.post("/scrape")
+async def scrape(url: str = Form(...)):
+
+    text = scrape_url(url)
+
+    chunks = chunk_text(text, url)
+
+    retriever.build_index(chunks)
+
+    return {"status": "webpage indexed"}
+
+
+@app.post("/ask")
+async def ask(question: str = Form(...)):
+
+    retrieved = retriever.query(question, k=25)
+
+    if not retrieved:
+        return {
+            "answer":"Upload a PDF first",
+            "chunks":[]
+        }
+
+    reranked = rerank(question, retrieved)
+
+    top_chunks = reranked[:2]
+
+    context = "\n\n".join(
+    c["text"][:900]
+    for c in top_chunks
+    )   
+    context = context.replace("\n"," ")
+
+    try:
+
+        answer = generate_answer(context, question)
+
+    except Exception as e:
+
+        print("LLM ERROR:", e)
+
+        answer = (
+            "⚠️ LLM unavailable. Showing best result:\n\n"
+            + top_chunks[0]["text"][:600]
+        )
+
+    return {
+        "answer":answer,
+        "chunks":top_chunks
+    }

requirements.txt

@@ -0,0 +1,17 @@
+fastapi
+uvicorn
+sentence-transformers
+faiss-cpu
+rank-bm25
+numpy
+pypdf
+requests
+beautifulsoup4
+torch
+transformers
+jinja2
+python-multipart
+transformers
+torch
+faiss-cpu
+numpy

reranker.py

@@ -0,0 +1,17 @@
+from sentence_transformers import CrossEncoder
+
+reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
+
+def rerank(question,chunks):
+
+    pairs=[[question,c["text"]] for c in chunks]
+
+    scores=reranker.predict(pairs)
+
+    ranked=sorted(
+        zip(scores,chunks),
+        key=lambda x:x[0],
+        reverse=True
+    )
+
+    return [c for _,c in ranked]

retrieval_colbert.py

@@ -0,0 +1,83 @@
+import torch
+import faiss
+import numpy as np
+
+from transformers import AutoTokenizer, AutoModel
+
+MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+model = AutoModel.from_pretrained(MODEL_NAME)
+
+class ColBERTRetriever:
+
+    def __init__(self):
+
+        self.chunks = []
+        self.doc_embeddings = []
+        self.index = None
+
+    # -----------------------------
+    # EMBED TEXT TOKENS
+    # -----------------------------
+
+    def embed(self, text):
+
+        inputs = tokenizer(
+            text,
+            return_tensors="pt",
+            truncation=True,
+            padding=True,
+            max_length=256
+        )
+
+        with torch.no_grad():
+            outputs = model(**inputs)
+
+        embeddings = outputs.last_hidden_state.squeeze(0)
+
+        return embeddings.numpy()
+
+    # -----------------------------
+    # BUILD INDEX
+    # -----------------------------
+
+    def build_index(self, chunks):
+
+        self.chunks = chunks
+        vectors = []
+
+        for c in chunks:
+
+            emb = self.embed(c["text"])
+            vectors.append(emb.mean(axis=0))
+
+        vectors = np.array(vectors).astype("float32")
+
+        dim = vectors.shape[1]
+
+        self.index = faiss.IndexFlatIP(dim)
+
+        self.index.add(vectors)
+
+    # -----------------------------
+    # QUERY
+    # -----------------------------
+
+    def query(self, question, k=20):
+
+        q_emb = self.embed(question)  # token embeddings
+        scores = []
+
+        for chunk in self.chunks:
+
+            d_emb = self.embed(chunk["text"])
+
+            sim = np.matmul(q_emb, d_emb.T)   # token similarity
+            score = sim.max(axis=1).sum()     # MaxSim
+
+            scores.append(score)
+
+        idx = np.argsort(scores)[::-1][:k]
+
+        return [self.chunks[i] for i in idx]

scraper.py

@@ -0,0 +1,20 @@
+import requests
+from bs4 import BeautifulSoup
+
+def scrape_url(url):
+
+    headers = {"User-Agent":"Mozilla/5.0"}
+
+    r = requests.get(url, headers=headers)
+
+    soup = BeautifulSoup(r.text,"html.parser")
+
+    elements = soup.find_all(["h1","h2","h3","p","li"])
+
+    text = " ".join(
+        el.get_text(strip=True)
+        for el in elements
+        if el.get_text(strip=True)
+    )
+
+    return text

templates/index.html

@@ -0,0 +1,323 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+
+<title>Scholar RAG Engine</title>
+
+<style>
+
+:root{
+    --bg:#f2f2f2;
+    --card:#ffffff;
+    --text:#111;
+    --accent:#2d6cdf;
+}
+
+.dark{
+    --bg:#0f172a;
+    --card:#1e293b;
+    --text:#e5e7eb;
+    --accent:#3b82f6;
+}
+
+body{
+    font-family: Arial;
+    background:var(--bg);
+    color:var(--text);
+    padding:40px;
+    transition:0.3s;
+}
+
+.container{
+    max-width:900px;
+    margin:auto;
+}
+
+.card{
+    background:var(--card);
+    padding:25px;
+    margin-bottom:25px;
+    border-radius:12px;
+    box-shadow:0 4px 14px rgba(0,0,0,0.1);
+}
+
+button{
+    padding:10px 20px;
+    background:var(--accent);
+    color:white;
+    border:none;
+    border-radius:6px;
+    cursor:pointer;
+}
+
+button:hover{
+    opacity:0.9;
+}
+
+input[type=text]{
+    width:100%;
+    padding:10px;
+    margin-top:10px;
+    margin-bottom:10px;
+    border-radius:6px;
+    border:1px solid #ccc;
+}
+
+details{
+    margin-top:10px;
+    background:#f7f7f7;
+    padding:10px;
+    border-radius:6px;
+}
+
+.dark details{
+    background:#334155;
+}
+
+summary{
+    cursor:pointer;
+    font-weight:bold;
+}
+
+.toggle{
+    float:right;
+}
+
+.status{
+    font-size:14px;
+    opacity:0.8;
+}
+
+/* LOADER */
+
+.loader{
+    display:none;
+    margin-top:10px;
+    font-size:14px;
+    color:var(--accent);
+}
+
+.spinner{
+    border:4px solid #f3f3f3;
+    border-top:4px solid var(--accent);
+    border-radius:50%;
+    width:18px;
+    height:18px;
+    animation:spin 1s linear infinite;
+    display:inline-block;
+    margin-right:8px;
+}
+
+@keyframes spin{
+    0%{transform:rotate(0deg)}
+    100%{transform:rotate(360deg)}
+}
+
+</style>
+
+</head>
+
+<body>
+
+<div class="container">
+
+<h1>
+📚 Scholar RAG Engine
+<button class="toggle" onclick="toggleMode()">🌙</button>
+</h1>
+
+<!-- PDF Upload -->
+
+<div class="card">
+
+<h2>Upload PDF</h2>
+
+<input type="file" id="pdf">
+
+<br><br>
+
+<button onclick="upload()">Upload & Index</button>
+
+<div id="uploadLoader" class="loader">
+<span class="spinner"></span> Indexing document...
+</div>
+
+<p id="uploadStatus" class="status"></p>
+
+</div>
+
+<!-- Website Scraper -->
+
+<div class="card">
+
+<h2>Scrape Website</h2>
+
+<input type="text" id="url" placeholder="Paste website URL">
+
+<button onclick="scrape()">Scrape & Index</button>
+
+<div id="scrapeLoader" class="loader">
+<span class="spinner"></span> Scraping website and indexing...
+</div>
+
+<p id="scrapeStatus" class="status"></p>
+
+</div>
+
+<!-- Ask Question -->
+
+<div class="card">
+
+<h2>Ask Question</h2>
+
+<input type="text" id="question" placeholder="Ask something from indexed documents">
+
+<button onclick="ask()">Ask</button>
+
+<div id="askLoader" class="loader">
+<span class="spinner"></span> Retrieving answer...
+</div>
+
+</div>
+
+<!-- Answer -->
+
+<div class="card">
+
+<h2>Answer</h2>
+
+<p id="answer">Answer will appear here</p>
+
+<br>
+
+<button onclick="toggleChunks()">Show Retrieved Chunks</button>
+
+<div id="chunks" style="display:none;margin-top:15px;"></div>
+
+</div>
+
+</div>
+
+<script>
+
+function toggleMode(){
+    document.body.classList.toggle("dark")
+}
+
+async function upload(){
+
+    let file=document.getElementById("pdf").files[0]
+
+    if(!file){
+        alert("Please select a PDF")
+        return
+    }
+
+    document.getElementById("uploadLoader").style.display="block"
+
+    let formData=new FormData()
+
+    formData.append("file",file)
+
+    let res=await fetch("/upload",{
+        method:"POST",
+        body:formData
+    })
+
+    let data=await res.json()
+
+    document.getElementById("uploadLoader").style.display="none"
+
+    document.getElementById("uploadStatus").innerText="Status: "+data.status
+}
+
+async function scrape(){
+
+    let url=document.getElementById("url").value
+
+    if(!url){
+        alert("Enter a URL")
+        return
+    }
+
+    document.getElementById("scrapeLoader").style.display="block"
+
+    let formData=new FormData()
+
+    formData.append("url",url)
+
+    let res=await fetch("/scrape",{
+        method:"POST",
+        body:formData
+    })
+
+    let data=await res.json()
+
+    document.getElementById("scrapeLoader").style.display="none"
+
+    document.getElementById("scrapeStatus").innerText="Status: "+data.status
+}
+
+async function ask(){
+
+    let question=document.getElementById("question").value
+
+    if(!question){
+        alert("Enter a question")
+        return
+    }
+
+    document.getElementById("askLoader").style.display="block"
+
+    let formData=new FormData()
+
+    formData.append("question",question)
+
+    let res=await fetch("/ask",{
+        method:"POST",
+        body:formData
+    })
+
+    let data=await res.json()
+
+    document.getElementById("askLoader").style.display="none"
+
+    document.getElementById("answer").innerText=data.answer
+
+    let chunkDiv=document.getElementById("chunks")
+
+    chunkDiv.innerHTML=""
+
+    if(data.chunks){
+
+        data.chunks.forEach((c,i)=>{
+
+            chunkDiv.innerHTML+=`
+            <details>
+                <summary>Chunk ${i+1} (${c.source})</summary>
+                <p>${c.text}</p>
+            </details>
+            `
+
+        })
+
+    }
+
+}
+
+function toggleChunks(){
+
+    let div=document.getElementById("chunks")
+
+    if(div.style.display==="none")
+        div.style.display="block"
+    else
+        div.style.display="none"
+
+}
+
+</script>
+
+</body>
+</html>