Upload 10 files

README.md

@@ -1,14 +1,33 @@
----
-title: GraphRAG Live
-emoji: 📊
-colorFrom: green
-colorTo: gray
-sdk: gradio
-sdk_version: 5.46.1
-app_file: app.py
-pinned: false
-license: mit
-short_description: Hybrid RAG with Graph + Vectors
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# GraphRAG-Live
+
+**Hybrid Retrieval-Augmented Generation (RAG) with Graph + Vectors.**
+
+This project shows how knowledge graphs (Neo4j Aura) and vector databases (Qdrant) can be combined with re-ranking heuristics to build a smarter, cheaper and more explainable RAG system.
+
+---
+
+## 🌟 Features
+- **Hybrid Retrieval:** Combines semantic search (Qdrant) with graph proximity scoring (Neo4j).  
+- **Dynamic Knowledge Injection:** Add new documents on the fly → pipeline updates instantly.  
+- **Evidence Subgraphs:** Each answer includes a small 2-hop evidence graph.  
+- **Metrics Dashboard:** Compare GraphRAG vs. baseline RAG on hit@10, nDCG@10, citation correctness.  
+- **Hosted Demo:** Deployed via Hugging Face Spaces (Gradio UI).  
+
+---
+
+## 🏗️ Architecture
+
+```text
+User Question
+      │
+      ▼
+[Qdrant: semantic chunks] + [Neo4j: graph proximity]
+      │
+      ▼
+Reranker (cosine + path proximity + freshness + degree)
+      │
+      ▼
+Answer Generator (OpenAI)
+      │
+      ▼
+Evidence Subgraph + Answer + Citations

app.py

@@ -0,0 +1,374 @@
+from fastapi import FastAPI
+from pydantic import BaseModel, Field
+from typing import List, Literal
+from datetime import datetime
+import os, json
+
+from text import chunk_text
+from vec import embed_and_upsert, search
+from kg import (
+    extract_and_insert,
+    get_subgraph,
+    compute_path_proximity,
+    compute_degree_norm,
+)
+from rerank import rerank_candidates
+from eval import evaluate
+from utils import compute_freshness
+
+from dotenv import load_dotenv
+from openai import OpenAI
+
+load_dotenv(override=True)
+
+key = os.environ.get("OPENAI_API_KEY", "").strip()
+client = OpenAI(api_key=key)
+
+app = FastAPI()
+
+
+# Schemas for Pydantic + structured output
+class DocInput(BaseModel):
+    text: str
+    source: str = "user"
+    timestamp: datetime = datetime.now()
+
+
+class QuestionInput(BaseModel):
+    question: str
+    w_cos: float = 0.60
+    w_path: float = 0.20
+    w_fresh: float = 0.15
+    w_deg: float = 0.05
+
+
+# LLM output requirement (enforceing this with JSON output + Pydantic)
+class LLMAnswer(BaseModel):
+    answer: str = Field(..., description="One-sentence final answer")
+    citations: List[str] = Field(
+        default_factory=list,
+        description="Evidence IDs like E1, E3 that support the answer",
+    )
+    graph_reasoning: str = Field(
+        "", description="How the graph helped, or 'Not used'"
+    )
+    confidence: Literal["High", "Medium", "Low"] = "Low"
+
+
+# Helpers for the explanation on the controls (weights)
+def _get_scores(c, w_cos, w_path, w_fresh, w_deg):
+    cos = float(c.get("cosine", c.get("cosine_sim", 0.0)) or 0.0)
+    pp = float(c.get("path_proximity", 0.0) or 0.0)
+    fr = float(c.get("freshness_decay", 0.0) or 0.0)
+    dg = float(c.get("degree_norm", 0.0) or 0.0)
+    final = w_cos * cos + w_path * pp + w_fresh * fr + w_deg * dg
+    return cos, pp, fr, dg, final
+
+
+def _build_knobs_breakdown(numbered, w_cos, w_path, w_fresh, w_deg):
+    """
+    Returns (knobs_line, knobs_explain) strings. Uses top 1 only and runner up if available.
+    """
+    if not numbered:
+        return "", ""
+
+    idx1, c1 = numbered[0]
+    cos1, pp1, fr1, dg1, fin1 = _get_scores(c1, w_cos, w_path, w_fresh, w_deg)
+
+    # Optional runner up
+    ru_piece, explain = "", ""
+    if len(numbered) > 1:
+        idx2, c2 = numbered[1]
+        cos2, pp2, fr2, dg2, fin2 = _get_scores(c2, w_cos, w_path, w_fresh, w_deg)
+        margin = fin1 - fin2
+        ru_piece = f"; Runner-up E{idx2}={fin2:.3f}; Margin={margin:+.3f}"
+
+        # Contribution of the deltas (weighted)
+        deltas = [
+            ("path", w_path * (pp1 - pp2), pp1, pp2, w_path),
+            ("freshness", w_fresh * (fr1 - fr2), fr1, fr2, w_fresh),
+            ("cosine", w_cos * (cos1 - cos2), cos1, cos2, w_cos),
+            ("degree", w_deg * (dg1 - dg2), dg1, dg2, w_deg),
+        ]
+        deltas.sort(key=lambda x: x[1], reverse=True)
+        # Pick top positive drivers
+        drivers = [f"{name} ({d:+.3f})" for name, d, *_ in deltas if d > 0.002][:3]
+        # A short natural language sentence
+        if drivers:
+            top_names = ", ".join(drivers)
+        else:
+            top_names = "mostly cosine similarity (others were negligible)"
+        explain = (
+            f"With weights (cos {w_cos:.2f}, path {w_path:.2f}, fresh {w_fresh:.2f}, deg {w_deg:.2f}), "
+            f"E{idx1} leads by {margin:+.3f}. Biggest lifts vs E{idx2}: {top_names}."
+        )
+    else:
+        # No runner up but sstill provide a brief note
+        explain = (
+            f"With weights (cos {w_cos:.2f}, path {w_path:.2f}, fresh {w_fresh:.2f}, deg {w_deg:.2f}), "
+            f"the top candidate E{idx1} scored {fin1:.3f}."
+        )
+
+    knobs_line = (
+        f"Weights→ cos {w_cos:.2f}, path {w_path:.2f}, fresh {w_fresh:.2f}, deg {w_deg:.2f}. "
+        f"E{idx1} final={fin1:.3f} = {w_cos:.2f}×{cos1:.3f} + {w_path:.2f}×{pp1:.3f} + "
+        f"{w_fresh:.2f}×{fr1:.3f} + {w_deg:.2f}×{dg1:.3f}{ru_piece}; Cosine-only(E{idx1})={cos1:.3f}."
+    )
+    return knobs_line, explain
+
+
+# API Endpoints
+@app.get("/metrics")
+def metrics_endpoint():
+    logs = []
+    try:
+        results = evaluate()
+        logs.append("✅ Ran evaluation set")
+        return {"status": "ok", "results": results, "logs": logs}
+    except Exception as e:
+        logs.append(f"⚠️ Metrics failed: {e}")
+        return {"status": "error", "logs": logs}
+
+
+@app.post("/add_doc")
+def add_doc_endpoint(doc: DocInput):
+    logs = ["📥 Received document"]
+    text, source, timestamp = doc.text, doc.source, doc.timestamp
+
+    # 1) Chunk
+    chunks = chunk_text(text)
+    logs.append(f"✂️ Chunked into {len(chunks)} pieces")
+
+    # 2) Embed + store
+    embed_and_upsert(chunks, source=source, timestamp=timestamp.isoformat())
+    logs.append(f"🧮 Embedded + stored in Qdrant (source={source}, ts={timestamp})")
+
+    # 3) Extract triples and feed to Neo4j
+    neo4j_logs = extract_and_insert(chunks, source=source, timestamp=str(timestamp))
+    logs.extend(neo4j_logs or ["🌐 No entities/relations extracted for Neo4j"])
+    return {"status": "ok", "logs": logs}
+
+
+@app.post("/ask")
+def ask_endpoint(query: QuestionInput):
+    logs = []
+    q = query.question
+    logs.append(f"❓ Received question: {q}")
+
+    # Retrieve
+    candidates = search(q, top_k=5)
+    logs.append(f"🔎 Retrieved {len(candidates)} from Qdrant")
+
+    # Graph aware features?? 
+    for c in candidates:
+        c["path_proximity"] = compute_path_proximity(q, c["chunk"])
+        c["degree_norm"] = compute_degree_norm(c["chunk"])
+        c["freshness_decay"] = compute_freshness(c.get("timestamp"))
+
+    # Rerank
+    reranked, rerank_logs = rerank_candidates(
+        candidates,
+        w_cos=query.w_cos,
+        w_path=query.w_path,
+        w_fresh=query.w_fresh,
+        w_deg=query.w_deg,
+    )
+    logs.append("📊 Applied graph-aware re-ranking")
+    logs.extend(rerank_logs)
+
+    # Evidence subgraph (≤2 hops)
+    triples = get_subgraph(q, source=None)
+    logs.append(f"🌐 Subgraph triples: {len(triples)}")
+
+    # Prepare evidence numbering for citations
+    numbered = [(i + 1, c) for i, c in enumerate(reranked)]
+    TOP_N = 2  # TODO -> expermient with more
+    reranked = reranked[:TOP_N]
+    numbered = [(i + 1, c) for i, c in enumerate(reranked)]
+    evidence_for_prompt = [f"[E{i}] {c['chunk']}" for i, c in numbered]
+    evidence_for_ui = [f"[E{i}] {c['chunk']}" for i, c in numbered]
+
+    knobs_line, knobs_explain = _build_knobs_breakdown(
+        numbered, query.w_cos, query.w_path, query.w_fresh, query.w_deg
+    )
+
+    # LLM answer (OpenAI, structured JSON -> Pydantic)
+    if reranked:
+        triples_text = "\n".join([f"({s}) -[{r}]-> ({o})" for s, r, o in triples])
+
+        # Schema friendly request
+        prompt = f"""
+You are a precise QA assistant that MUST use BOTH the retrieved evidence and the graph triples.
+
+Question:
+{q}
+
+Retrieved Evidence (ranked by importance, highest first):
+{chr(10).join(evidence_for_prompt)}
+
+Knowledge Graph Triples:
+{triples_text}
+
+Instructions:
+- E1 is the most relevant, E2 is second-most, and so on.
+- Prefer evidence with a lower number if multiple sources conflict.
+- If supported, produce a single-sentence answer.
+- Cite supporting evidence IDs (e.g., E1, E2).
+- If the graph helped, say how; else "Not used".
+- If not supported, return "I don’t know..." with Low confidence.
+
+Return ONLY a JSON object matching this schema:
+{{
+  "answer": "string",
+  "citations": ["E1","E2"],
+  "graph_reasoning": "string",
+  "confidence": "High|Medium|Low"
+}}
+""".strip()
+
+        logs.append("📝 Built prompt with evidence + graph")
+        try:
+            comp = client.chat.completions.create(
+                model="gpt-4o-mini",
+                messages=[
+                    {"role": "system", "content": "Respond ONLY with a JSON object."},
+                    {"role": "user", "content": prompt},
+                ],
+                # Ensures valid JSON
+                response_format={"type": "json_object"},
+                temperature=0,
+                max_tokens=300,
+            )
+            raw_json = comp.choices[0].message.content or "{}"
+            data = json.loads(raw_json)
+
+            # Validate and normalize with Pydantic
+            parsed = LLMAnswer.model_validate(data)
+
+            # Build display string for the UI card
+            citations_txt = ", ".join(parsed.citations) if parsed.citations else "None"
+            answer_text = (
+                f"{parsed.answer}\n"
+                f"Citations: {citations_txt}\n"
+                f"Graph reasoning: {parsed.graph_reasoning or '—'}\n"
+                f"Confidence: {parsed.confidence}\n"
+                f"Knobs: {knobs_line or '—'}\n"
+                f"Knobs explain: {knobs_explain or '—'}"
+            )
+
+            answer = answer_text
+            logs.append("🤖 Called OpenAI")
+            logs.append("🧠 Generated final answer")
+        except Exception as e:
+            top_chunk = reranked[0]["chunk"] if reranked else "No evidence"
+            answer = (
+                f"Based on evidence: {top_chunk}\n"
+                f"Citations: None\n"
+                f"Graph reasoning: Not used\n"
+                f"Confidence: Low\n"
+                f"Knobs: {knobs_line or '—'}\n"
+                f"Knobs explain: {knobs_explain or '—'}"
+            )
+            logs.append(f"⚠️ OpenAI failed, fallback to stub ({e})")
+    else:
+        answer = (
+            "No evidence found.\n"
+            "Citations: None\n"
+            "Graph reasoning: Not used\n"
+            "Confidence: Low\n"
+            f"Knobs: {knobs_line or '—'}\n"
+            f"Knobs explain: {knobs_explain or '—'}"
+        )
+        evidence_for_ui = []
+        logs.append("⚠️ No evidence, answer is empty")
+
+    # Build D3 JSON
+    node_map = {}
+    links = []
+    for s, r, o in triples:
+        node_map.setdefault(s, {"id": s})
+        node_map.setdefault(o, {"id": o})
+        links.append({"source": s, "target": o, "label": r})
+    subgraph_json = {"nodes": list(node_map.values()), "links": links}
+
+    # Server side SVG fallback in case D3 fails to render
+    import networkx as nx
+
+    G = nx.DiGraph()
+    for s, r, o in triples:
+        G.add_node(s)
+        G.add_node(o)
+        G.add_edge(s, o, label=r)
+
+    pos = nx.spring_layout(G, seed=42)
+    width, height, pad = 720, 420, 40
+    xs = [p[0] for p in pos.values()] or [0.0]
+    ys = [p[1] for p in pos.values()] or [0.0]
+    minx, maxx = min(xs), max(xs)
+    miny, maxy = min(ys), max(ys)
+    rangex = (maxx - minx) or 1.0
+    rangey = (maxy - miny) or 1.0
+
+    def sx(x): return pad + (x - minx) / rangex * (width - 2 * pad)
+    def sy(y): return pad + (y - miny) / rangey * (height - 2 * pad)
+
+    parts = []
+    parts.append(
+        f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
+        f'xmlns="http://www.w3.org/2000/svg">'
+    )
+    parts.append(
+        """
+    <defs>
+      <marker id="arrow" markerUnits="strokeWidth" markerWidth="10" markerHeight="8"
+              viewBox="0 0 10 8" refX="10" refY="4" orient="auto">
+        <path d="M0 0 L10 4 L0 8 z" fill="#999"/>
+      </marker>
+      <style>
+        .edge { stroke:#999; stroke-width:1.5; }
+        .nodelabel { font:12px sans-serif; fill:#ddd; }
+        .edgelabel { font:10px sans-serif; fill:#bbb; }
+        .node { fill:#69b3a2; stroke:#2dd4bf; stroke-width:1; }
+      </style>
+    </defs>
+    """
+    )
+
+    for u, v, data in G.edges(data=True):
+        x1, y1 = sx(pos[u][0]), sy(pos[u][1])
+        x2, y2 = sx(pos[v][0]), sy(pos[v][1])
+        parts.append(
+            f'<line class="edge" x1="{x1:.1f}" y1="{y1:.1f}" '
+            f'x2="{x2:.1f}" y2="{y2:.1f}" marker-end="url(#arrow)"/>'
+        )
+        mx, my = (x1 + x2) / 2.0, (y1 + y2) / 2.0
+        lbl = (data.get("label") or "").replace("&", "&amp;").replace("<", "&lt;")
+        parts.append(
+            f'<text class="edgelabel" x="{mx:.1f}" y="{my:.1f}" text-anchor="middle">{lbl}</text>'
+        )
+
+    for n in G.nodes():
+        x, y = sx(pos[n][0]), sy(pos[n][1])
+        node_txt = str(n).replace("&", "&amp;").replace("<", "&lt;")
+        r = max(16, len(node_txt) * 4)
+        parts.append(f'<circle class="node" cx="{x:.1f}" cy="{y:.1f}" r="{r}"/>')
+        parts.append(
+            f'<text class="nodelabel" x="{x:.1f}" y="{y + r + 14:.1f}" text-anchor="middle">{node_txt}</text>'
+        )
+    parts.append("</svg>")
+    subgraph_svg = "".join(parts)
+
+    logs.append(f"📦 Subgraph JSON dump: {subgraph_json}")
+
+    return {
+        "answer": answer,
+        "evidence": evidence_for_ui,
+        "subgraph_svg": subgraph_svg,     # fallback
+        "subgraph_json": subgraph_json,   # for D3 in UI
+        "logs": logs,
+    }
+
+
+@app.get("/healthz")
+def healthz():
+    return {"ok": True}

eval.py

@@ -0,0 +1,71 @@
+import time
+from vec import search
+from rerank import rerank_candidates
+from kg import compute_path_proximity, compute_degree_norm
+from utils import compute_freshness
+
+# A toy dataset: {question: [expected substrings]}
+EVAL_SET = [
+    {"q": "Who founded OpenAI?", "answers": ["Elon Musk", "Sam Altman"]},
+    {"q": "What did OpenAI release?", "answers": ["GPT-4o", "Whisper", "SORA"]},
+    {"q": "What did Google acquire?", "answers": ["YouTube", "Instagram"]},
+]
+
+def evaluate():
+    baseline_hits, hybrid_hits = [], []
+    baseline_ndcg, hybrid_ndcg = [], []
+    citation_correctness = []
+    latencies = []
+
+    for item in EVAL_SET:
+        q, golds = item["q"], item["answers"]
+
+        # Baseline (cosine only)
+        start = time.time()
+        baseline = search(q, top_k=10)
+        latencies.append(time.time() - start)
+
+        # Did we hit a gold in top-10?
+        hit = any(any(g.lower() in c["chunk"].lower() for g in golds) for c in baseline)
+        baseline_hits.append(1 if hit else 0)
+
+        # nDCG@10
+        scores = []
+        for rank, c in enumerate(baseline, 1):
+            rel = 1 if any(g.lower() in c["chunk"].lower() for g in golds) else 0
+            if rel:
+                scores.append(1 / (rank))
+        baseline_ndcg.append(sum(scores))
+
+        # Hybrid (cosine + path + freshness + degree)
+        for c in baseline:
+            c["path_proximity"] = compute_path_proximity(q, c["chunk"])
+            c["degree_norm"] = compute_degree_norm(c["chunk"])
+            c["freshness_decay"] = compute_freshness(c.get("timestamp"))
+
+        reranked, _ = rerank_candidates(baseline)
+        hit = any(any(g.lower() in c["chunk"].lower() for g in golds) for c in reranked[:10])
+        hybrid_hits.append(1 if hit else 0)
+
+        scores = []
+        for rank, c in enumerate(reranked, 1):
+            rel = 1 if any(g.lower() in c["chunk"].lower() for g in golds) else 0
+            if rel:
+                scores.append(1 / (rank))
+        hybrid_ndcg.append(sum(scores))
+
+        # TODO -> Citation correctness
+        citation_correctness.append(1)
+
+    return {
+        "baseline": {
+            "hit@10": sum(baseline_hits)/len(baseline_hits),
+            "nDCG@10": sum(baseline_ndcg)/len(baseline_ndcg),
+        },
+        "hybrid": {
+            "hit@10": sum(hybrid_hits)/len(hybrid_hits),
+            "nDCG@10": sum(hybrid_ndcg)/len(hybrid_ndcg),
+        },
+        "citation_correctness": sum(citation_correctness)/len(citation_correctness),
+        "avg_latency_sec": sum(latencies)/len(latencies),
+    }

kg.py

@@ -0,0 +1,315 @@
+import os, re, json
+from dotenv import load_dotenv
+from neo4j import GraphDatabase
+import spacy
+
+try:
+    from groq import Groq
+except Exception:
+    Groq = None
+
+load_dotenv()
+nlp = spacy.load("en_core_web_sm")
+
+uri = os.getenv("NEO4J_URI")
+user = os.getenv("NEO4J_USER")
+password = os.getenv("NEO4J_PASS")
+database = os.getenv("NEO4J_DATABASE", "neo4j")
+
+# Options for "spacy" and "groq"
+KG_EXTRACTOR = os.getenv("KG_EXTRACTOR", "spacy").strip().lower()
+
+# Groq config (to extract the triplets)
+GROQ_API_KEY = os.getenv("GROQ_API_KEY", "").strip()
+GROQ_MODEL = os.getenv("GROQ_MODEL", "openai/gpt-oss-20b").strip()
+_groq = Groq(api_key=GROQ_API_KEY) if (Groq and GROQ_API_KEY) else None
+
+driver = GraphDatabase.driver(uri, auth=(user, password))
+
+ORG_HINTS = {
+    "inc","corp","corporation","ltd","llc","bank","securities","university",
+    "labs","institute","tech","technologies","systems","solutions","group"
+}
+
+# scoring helpers
+def compute_path_proximity(question, chunk):
+    q_doc = nlp(question)
+    c_doc = nlp(chunk)
+    q_ents = {ent.text.lower() for ent in q_doc.ents}
+    c_ents = {ent.text.lower() for ent in c_doc.ents}
+    return 1.0 if q_ents & c_ents else 0.0
+
+def compute_degree_norm(chunk):
+    doc = nlp(chunk)
+    ents = [ent.text for ent in doc.ents]
+    if not ents:
+        return 0.0
+    degrees = []
+    with driver.session(database=database) as session:
+        for e in ents:
+            res = session.run("""
+                MATCH (n {name_lc:$name_lc})
+                RETURN count { (n)--() } AS deg
+                LIMIT 1
+            """, name_lc=normalize_key(e))
+            rec = res.single()
+            if rec and rec["deg"] is not None:
+                degrees.append(rec["deg"])
+    if not degrees:
+        return 0.0
+    return 1.0 / (1 + sum(degrees) / len(degrees))
+
+# label + text helpers
+def get_label(ent_label: str) -> str:
+    mapping = {
+        "PERSON": "Person","ORG": "Org","GPE": "Location","NORP": "Group",
+        "FAC": "Facility","LOC": "Location","PRODUCT": "Product","EVENT": "Event",
+        "WORK_OF_ART": "Work","LAW": "Law","LANGUAGE": "Language","DATE": "Date",
+        "TIME": "Time","PERCENT": "Percent","MONEY": "Money","QUANTITY": "Quantity",
+        "ORDINAL": "Ordinal","CARDINAL": "Number",
+    }
+    return mapping.get(ent_label, "Entity")
+
+def normalize_key(text: str) -> str:
+    return re.sub(r"\s+", " ", text.strip()).lower()
+
+def entity_label_for_text(text: str, doc) -> str:
+    for ent in doc.ents:
+        if ent.text == text:
+            return get_label(ent.label_)
+    words = {w.lower() for w in text.split()}
+    if words & ORG_HINTS:
+        return "Org"
+    if text and text[0].isupper() and " " not in text:
+        return "Person"
+    return "Entity"
+
+def span_text_for_token(tok, doc) -> str:
+    for ent in doc.ents:
+        if ent.start <= tok.i < ent.end:
+            return ent.text
+    left = [w for w in tok.lefts if w.dep_ in ("compound","amod","flat","nmod")]
+    right = [w for w in tok.rights if w.dep_ in ("compound","flat","nmod")]
+    tokens = sorted([*left, tok, *right], key=lambda t: t.i)
+    return " ".join(t.text for t in tokens if t.pos_ != "PUNCT")
+
+def subjects_for_verb(v):
+    subs = [w for w in v.lefts if w.dep_ in ("nsubj","nsubjpass","csubj")]
+    if not subs and v.dep_ == "conj":
+        subs = subjects_for_verb(v.head)
+    out = []
+    for s in subs:
+        out.append(s)
+        out.extend(list(s.conjuncts))
+    return out
+
+def objects_for_verb(v):
+    objs = [w for w in v.rights if w.dep_ in ("dobj","attr","pobj","dative","oprd")]
+    for prep in [w for w in v.rights if w.dep_ == "prep"]:
+        objs.extend([w for w in prep.rights if w.dep_ == "pobj"])
+    out = []
+    for o in objs:
+        out.append(o)
+        out.extend(list(o.conjuncts))
+    return out
+
+# spaCy extractor
+def _extract_triples_spacy(text: str):
+    triples = []
+    doc = nlp(text)
+    for tok in doc:
+        if tok.pos_ == "VERB":
+            subs = subjects_for_verb(tok)
+            objs = objects_for_verb(tok)
+            if not subs or not objs:
+                continue
+            rel = tok.lemma_.upper()
+            for s in subs:
+                s_text = span_text_for_token(s, doc)
+                s_label = entity_label_for_text(s_text, doc)
+                for o in objs:
+                    o_text = span_text_for_token(o, doc)
+                    o_label = entity_label_for_text(o_text, doc)
+                    triples.append({
+                        "subject": s_text, "subject_label": s_label,
+                        "relation": rel,
+                        "object": o_text, "object_label": o_label
+                    })
+    return triples
+
+# Groq extractor (structured)
+_GROQ_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "triples": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "subject": {"type": "string"},
+                    "subject_label": {"type": "string"},
+                    "relation": {"type": "string"},
+                    "object": {"type": "string"},
+                    "object_label": {"type": "string"},
+                },
+                "required": ["subject", "relation", "object"]
+            }
+        }
+    },
+    "required": ["triples"]
+}
+
+_ALLOWED_LABELS = (
+    "Person","Org","Location","Group","Facility","Product","Event","Work",
+    "Law","Language","Date","Time","Percent","Money","Quantity","Ordinal",
+    "Number","Entity"
+)
+
+def _extract_triples_groq(text: str):
+    if not _groq:
+        raise RuntimeError("Groq not configured. Set GROQ_API_KEY or use KG_EXTRACTOR=spacy.")
+    prompt = f"""
+Extract concise subject-RELATION-object triples from the text.
+
+Rules:
+- Use a SINGLE UPPERCASE token for relation (e.g., ACQUIRE, FOUND, PARTNER_WITH).
+- Provide subject_label and object_label using this set: {_ALLOWED_LABELS}.
+- Merge duplicates; at most 8 triples per chunk.
+- Return ONLY JSON matching this schema:
+
+{json.dumps(_GROQ_SCHEMA, indent=2)}
+
+Text:
+\"\"\"{text}\"\"\"
+"""
+    resp = _groq.chat.completions.create(
+        model=GROQ_MODEL,
+        messages=[
+            {"role": "system", "content": "You are an information extractor. Output strictly valid JSON."},
+            {"role": "user", "content": prompt},
+        ],
+        temperature=0,
+        max_tokens=600,        
+        response_format={"type": "json_object"},
+    )
+    raw = resp.choices[0].message.content or "{}"
+    try:
+        data = json.loads(raw)
+    except Exception:        
+        start = raw.find("{"); end = raw.rfind("}")
+        data = json.loads(raw[start:end+1]) if start != -1 and end != -1 else {"triples": []}
+    triples = data.get("triples", [])    
+    out = []
+    for t in triples:
+        subj = (t.get("subject") or "").strip()
+        obj  = (t.get("object")  or "").strip()
+        rel  = (t.get("relation") or "").strip().upper().replace(" ", "_")
+        if not subj or not obj or not rel:
+            continue
+        sl = t.get("subject_label") or "Entity"
+        ol = t.get("object_label") or "Entity"
+        if sl not in _ALLOWED_LABELS: sl = "Entity"
+        if ol not in _ALLOWED_LABELS: ol = "Entity"
+        out.append({"subject": subj, "subject_label": sl, "relation": rel,
+                    "object": obj, "object_label": ol})
+    return out
+
+# Inserts
+def _insert_triples(triples, source: str, timestamp: str, logs: list):
+    if not triples:
+        return
+    with driver.session(database=database) as session:
+        for t in triples:
+            s_text, s_label = t["subject"], t["subject_label"]
+            o_text, o_label = t["object"], t["object_label"]
+            rel = t["relation"]
+            cypher = f"""
+            MERGE (a:{s_label} {{name_lc:$a_key}})
+              ON CREATE SET a.name = $a_name
+            MERGE (b:{o_label} {{name_lc:$b_key}})
+              ON CREATE SET b.name = $b_name
+            MERGE (a)-[r:{rel}]->(b)
+              ON CREATE SET r.source=$source, r.ts=$ts
+              SET r.source=$source, r.ts=$ts
+            """
+            session.run(
+                cypher,
+                a_key=normalize_key(s_text), a_name=s_text,
+                b_key=normalize_key(o_text), b_name=o_text,
+                source=source, ts=timestamp
+            )
+            logs.append(f"🌐 Inserted ({s_text}:{s_label})-[:{rel}]->({o_text}:{o_label}) [src={source}, ts={timestamp}]")
+
+
+# Public API used by app.py
+def extract_and_insert(chunks, source="user", timestamp=None):
+    """
+    Extract triples (Groq or spaCy) and insert into Neo4j with source and timestamp.
+    """
+    from datetime import datetime
+    if timestamp is None:
+        timestamp = datetime.now().isoformat()
+
+    logs = []
+    use_groq = (KG_EXTRACTOR == "groq") and _groq is not None
+    for chunk in chunks:
+        triples = []
+        if use_groq:
+            try:
+                triples = _extract_triples_groq(chunk)
+                logs.append(f"🤝 Groq extracted {len(triples)} triples")
+            except Exception as e:
+                logs.append(f"⚠️ Groq extraction failed: {e}. Falling back to spaCy.")
+        if not triples:
+            triples = _extract_triples_spacy(chunk)
+            logs.append(f"🧠 spaCy extracted {len(triples)} triples")
+        _insert_triples(triples, source=source, timestamp=timestamp, logs=logs)
+    return logs
+
+def test_connection():
+    with driver.session(database=database) as session:
+        msg = session.run("RETURN 'Connected to Neo4j!' AS msg").single()
+        print(msg["msg"])
+
+def get_subgraph(question: str, source: str | None = None, limit: int = 24, evidence_chunks: list[str] | None = None):
+    """
+    Subgraph search by using NER (spaCy) and words from evidence chunks.
+    """
+    results = set()
+    doc = nlp(question)
+    entities = [ent.text for ent in doc.ents] or [t.text for t in doc if t.pos_ in ("PROPN","NOUN")]
+
+    if evidence_chunks:
+        for chunk in evidence_chunks:
+            c_doc = nlp(chunk)
+            entities.extend([ent.text for ent in c_doc.ents])
+
+    # Normalize + deduplicate
+    entities = list({normalize_key(e) for e in entities if e.strip()})
+
+    with driver.session(database=database) as session:
+        for e in entities:
+            cypher = """
+            MATCH (a)-[r]-(b)
+            WHERE (
+                (a.name_lc IS NOT NULL AND a.name_lc CONTAINS $k) OR
+                (b.name_lc IS NOT NULL AND b.name_lc CONTAINS $k) OR
+                toLower(a.name) CONTAINS $k OR
+                toLower(b.name) CONTAINS $k
+            )
+            """ + (" AND r.source = $source " if source else "") + """
+            RETURN DISTINCT a, type(r) AS rel, b
+            LIMIT $limit
+            """
+            params = {"k": e, "limit": limit}
+            if source:
+                params["source"] = source
+
+            for rec in session.run(cypher, **params):
+                a, rel, b = rec["a"], rec["rel"], rec["b"]
+                results.add((
+                    a.get("name", a.get("name_lc","")),
+                    rel,
+                    b.get("name", b.get("name_lc",""))
+                ))
+    return list(results)

requirements.txt

@@ -0,0 +1,12 @@
+gradio>=4.45.0
+fastapi==0.111.0
+uvicorn==0.30.1
+neo4j==5.23.0
+qdrant-client==1.9.1
+sentence-transformers==2.7.0
+openai>=1.40.2
+httpx>=0.27.2
+python-dotenv==1.0.1
+spacy==3.8.7
+groq
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl

rerank.py

@@ -0,0 +1,30 @@
+def rerank_candidates(candidates, w_cos=0.60, w_path=0.20, w_fresh=0.15, w_deg=0.05):
+    """
+    Rerank chunks with a hybrid scoring formula.
+    Weights are configurable from the ui.
+    """
+    reranked = []
+    logs = []
+
+    for idx, c in enumerate(candidates, 1):
+        score = (
+            w_cos * c.get("cosine", 0) +
+            w_path * c.get("path_proximity", 0) +
+            w_fresh * c.get("freshness_decay", 0) +
+            w_deg * c.get("degree_norm", 0)
+        )
+        c["final_score"] = score
+        reranked.append(c)
+
+        logs.append(
+            f"Candidate {idx}: "
+            f"cosine={c.get('cosine',0):.3f}, "
+            f"path={c.get('path_proximity',0):.3f}, "
+            f"freshness={c.get('freshness_decay',0):.3f}, "
+            f"degree={c.get('degree_norm',0):.3f} "
+            f"→ final={score:.3f}"
+        )
+
+    reranked.sort(key=lambda x: x["final_score"], reverse=True)
+    return reranked, logs
+

text.py

@@ -0,0 +1,13 @@
+def chunk_text(text: str, chunk_size : int = 200, overlap: int = 50) -> str:
+    """
+    Here, we will break the text into overlapping chunks and then feed
+    them to the embedding pipeline
+    """
+    chunks = []
+    start = 0
+    while start < len(text):
+        end = start + chunk_size
+        curr_chunk = text[start:end]
+        chunks.append(curr_chunk)
+        start += chunk_size - overlap # we need some overlap between the chunks
+    return chunks

ui.py

@@ -0,0 +1,406 @@
+import gradio as gr
+import requests
+import html
+import os
+from datetime import date as _date
+
+API_URL = os.getenv("API_URL", "http://127.0.0.1:8000")
+
+# Helpers
+def _parse_answer_sections(answer_text: str):
+    lines = [l.strip() for l in (answer_text or "").splitlines() if l.strip()]
+    out = {
+        "main": "",
+        "citations": "",
+        "graph_reasoning": "",
+        "confidence": "",
+        "knobs": "",
+        "knobs_explain": "",
+    }
+    main_parts = []
+    for ln in lines:
+        ll = ln.lower()
+        if ll.startswith("citations:"):
+            out["citations"] = ln.split(":", 1)[1].strip()
+        elif ll.startswith("graph reasoning:") or ll.startswith("graphreasoning:"):
+            out["graph_reasoning"] = ln.split(":", 1)[1].strip()
+        elif ll.startswith("confidence:"):
+            out["confidence"] = ln.split(":", 1)[1].strip()
+        elif ll.startswith("knobs explain:"):
+            out["knobs_explain"] = ln.split(":", 1)[1].strip()
+        elif ll.startswith("knobs:"):
+            out["knobs"] = ln.split(":", 1)[1].strip()
+        else:
+            main_parts.append(ln)
+    out["main"] = " ".join(main_parts).strip() or (answer_text or "").strip()
+    return out
+
+
+def _confidence_class(conf: str) -> str:
+    c = (conf or "").strip().lower()
+    if c.startswith("high"):
+        return "badge-high"
+    if c.startswith("medium"):
+        return "badge-medium"
+    if c.startswith("low"):
+        return "badge-low"
+    return "badge-none"
+
+
+def _render_answer_card(answer_text: str) -> str:
+    sec = _parse_answer_sections(answer_text)
+    conf_cls = _confidence_class(sec["confidence"])
+    main = html.escape(sec["main"])
+    citations = html.escape(sec["citations"] or "None")
+    greason = html.escape(sec["graph_reasoning"] or "—")
+    conf = html.escape(sec["confidence"] or "—")
+    knobs = html.escape(sec["knobs"] or "—")
+    knobs_explain = html.escape(sec["knobs_explain"] or "—")
+    return f"""
+    <div class="card">
+        <div class="card-title">Answer</div>
+        <div class="answer">{main}</div>
+        <div class="meta">
+            <span class="badge {conf_cls}">{conf}</span>
+        </div>
+        <div class="sub"><b>Citations:</b> {citations}</div>
+        <div class="sub"><b>Graph reasoning:</b> {greason}</div>
+        <div class="sub"><b>Knobs effect:</b> {knobs}</div>
+        <div class="sub"><b>Knobs explain:</b> {knobs_explain}</div>
+    </div>
+    """
+
+
+def _render_evidence_markdown(evidence_list):
+    if not evidence_list:
+        return "_No evidence returned._"
+    lines = []
+    for i, chunk in enumerate(evidence_list, 1):
+        chunk = chunk.strip()
+        lines.append(f"**E{i}.** {chunk}")
+    return "\n\n".join(lines)
+
+
+def _wrap_svg(svg: str) -> str:
+    if not svg or "<svg" not in svg:
+        return "<div class='graph-empty'>No graph</div>"
+    return f"""<div class="graph-wrap">{svg}</div>"""
+
+
+def metrics_ui():
+    resp = requests.get(f"{API_URL}/metrics")
+    try:
+        j = resp.json()
+        if j.get("status") != "ok":
+            return f"Error: {j}"
+        r = j["results"]
+        return f"""
+### 📊 Evaluation Results
+
+**Baseline (cosine-only)**
+- hit@10: {r['baseline']['hit@10']:.2f}
+- nDCG@10: {r['baseline']['nDCG@10']:.2f}
+
+**Hybrid (GraphRAG)**
+- hit@10: {r['hybrid']['hit@10']:.2f}
+- nDCG@10: {r['hybrid']['nDCG@10']:.2f}
+
+**Other**
+- Citation correctness: {r['citation_correctness']:.2f}
+- Avg latency (s): {r['avg_latency_sec']:.2f}
+"""
+    except Exception as e:
+        return f"Error: {e}\nRaw: {resp.text[:500]}"
+
+
+def add_doc_ui(text, source="user", date_val=None, time_val=None):
+    payload = {"text": text, "source": source}
+
+    # Build ISO timestamp if a date was picked
+    ts_iso = ""
+    if date_val:        
+        if isinstance(date_val, _date):
+            dstr = date_val.isoformat()
+        else:
+            dstr = str(date_val)
+        
+        tstr = (time_val or "00:00").strip()
+        if len(tstr) == 5:   # HH:MM -> add seconds
+            tstr = f"{tstr}:00"
+        ts_iso = f"{dstr}T{tstr}Z"
+
+    if ts_iso:
+        payload["timestamp"] = ts_iso
+
+    resp = requests.post(f"{API_URL}/add_doc", json=payload)
+    try:
+        j = resp.json()
+        return "\n".join(j.get("logs", [])) or "No logs."
+    except Exception as e:
+        return f"Error: {e}\nRaw response: {resp.text[:500]}"
+
+
+def ask_ui(question, w_cos, w_path, w_fresh, w_deg):
+    payload = {
+        "question": question,
+        "w_cos": w_cos,
+        "w_path": w_path,
+        "w_fresh": w_fresh,
+        "w_deg": w_deg,
+    }
+    resp = requests.post(f"{API_URL}/ask", json=payload)
+    try:
+        j = resp.json()
+    except Exception as e:
+        err = f"Error: {e}\nRaw response: {resp.text[:500]}"
+        return (
+            _render_answer_card("I don’t know based on the given evidence.\nConfidence: Low"),
+            "_No evidence returned._",
+            err,
+            "<div id='graph' style='height:420px'></div>",
+            {},
+        )
+
+    answer_html = _render_answer_card(j.get("answer", ""))
+    evidence_md = _render_evidence_markdown(j.get("evidence", []))
+    logs_txt = "\n".join(j.get("logs", [])) or "No logs."
+
+    # D3 container but if no data fall back to server SVG
+    graph_json = j.get("subgraph_json", {})
+    if graph_json and graph_json.get("nodes"):
+        graph_html_value = "<div id='graph' style='height:420px'></div>"
+    else:
+        graph_html_value = _wrap_svg(j.get("subgraph_svg", ""))
+
+    return (answer_html, evidence_md, logs_txt, graph_html_value, graph_json)
+
+
+# UI
+with gr.Blocks(
+    css="""
+/* Layout & theme */
+body { background: #0b0f14; color: #e6edf3; }
+.gradio-container { max-width: 1180px !important; }
+.section-title { font-size: 22px; font-weight: 700; margin: 6px 0 12px; }
+
+/* Cards */
+.card { background: #0f1720; border: 1px solid #1f2a36; border-radius: 14px; padding: 14px; }
+.card-title { font-size: 16px; letter-spacing: .3px; color: #9fb3c8; margin-bottom: 8px; text-transform: uppercase; }
+.answer { font-size: 18px; line-height: 1.5; margin-bottom: 8px; }
+.sub { color: #a8b3bf; margin-top: 6px; font-size: 14px; }
+
+/* Badges */
+.badge { padding: 3px 10px; border-radius: 999px; font-size: 12px; font-weight: 700; display: inline-block; }
+.badge-high { background: #12391a; color: #6ee787; border: 1px solid #285f36; }
+.badge-medium { background: #3a2b13; color: #ffd277; border: 1px solid #6b4e1f; }
+.badge-low { background: #3b1616; color: #ff9492; border: 1px solid #6b2020; }
+.badge-none { background: #223; color: #9fb3c8; border: 1px solid #334; }
+
+/* Graph */
+.graph-wrap { background: #0f1720; border: 1px solid #1f2a36; border-radius: 14px;
+               padding: 12px; height: 460px; overflow: auto; }
+.graph-empty { color: #9fb3c8; font-style: italic; padding: 16px; }
+
+/* Logs */
+#logs-box textarea {
+  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace !important;
+  max-height: 280px !important;
+  overflow-y: auto !important;
+}
+"""
+) as demo:
+    gr.Markdown("### 🚀 GraphRAG — Live Demo")
+
+    with gr.Tab("Add Document"):
+        with gr.Row():
+            with gr.Column(scale=3):
+                text_in = gr.Textbox(
+                    label="Document",
+                    lines=10,
+                    placeholder="Paste text to inject into Graph + Vector DB…",
+                )
+            with gr.Column(scale=1):
+                source_in = gr.Textbox(label="Source", value="user")
+
+                if hasattr(gr, "Date"):
+                    ts_date = gr.Date(label="Date (optional)")
+                else:
+                    ts_date = gr.Textbox(label="Date (YYYY-MM-DD, optional)")
+
+                if hasattr(gr, "Time"):
+                    ts_time = gr.Time(label="Time (optional)", value="00:00")
+                else:
+                    ts_time = gr.Textbox(label="Time (HH:MM, optional)", value="00:00")
+
+                add_btn = gr.Button("Add Doc", variant="primary")
+                add_logs = gr.Textbox(label="Ingestion Logs", lines=14, elem_id="logs-box")
+
+                add_btn.click(
+                    add_doc_ui,
+                    inputs=[text_in, source_in, ts_date, ts_time],
+                    outputs=add_logs
+                )
+
+
+    with gr.Tab("Ask Question"):
+        with gr.Row():
+            q_in = gr.Textbox(
+                label="Question", placeholder="e.g., Who acquired Instagram?"
+            )
+            ask_btn = gr.Button("Ask", variant="primary")
+
+        with gr.Accordion("Rerank Weights", open=False):
+            w_cos = gr.Slider(0, 1, value=0.60, step=0.05, label="Cosine weight")
+            w_path = gr.Slider(0, 1, value=0.20, step=0.05, label="Path proximity weight")
+            w_fresh = gr.Slider(0, 1, value=0.15, step=0.05, label="Freshness weight")
+            w_deg = gr.Slider(0, 1, value=0.05, step=0.05, label="Degree norm weight")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.Markdown("<div class='section-title'>Answer</div>")
+                ans_html = gr.HTML(value=_render_answer_card("Ask something to see results."))
+
+                evid = gr.Accordion("Evidence (ranked)", open=True)
+                with evid:
+                    evid_md = gr.Markdown()
+
+                logs = gr.Accordion("Debug logs", open=False)
+                with logs:
+                    logs_txt = gr.Textbox(lines=14, elem_id="logs-box")
+
+            with gr.Column(scale=1):
+                gr.Markdown("<div class='section-title'>Evidence Graph</div>")
+                graph_html = gr.HTML(value="<div id='graph' style='height:600px'></div>")
+                graph_data = gr.JSON(label="graph-data", visible=False)
+
+        ask_btn.click(
+            ask_ui,
+            inputs=[q_in, w_cos, w_path, w_fresh, w_deg],
+            outputs=[ans_html, evid_md, logs_txt, graph_html, graph_data],
+        )
+
+    with gr.Tab("Metrics"):
+        metrics_btn = gr.Button("Run Evaluation", variant="primary")
+        metrics_out = gr.Markdown("Click run to evaluate baseline vs hybrid.")
+        metrics_btn.click(metrics_ui, inputs=[], outputs=metrics_out)
+
+    # D3 rendering for the knowledge graph
+    DRAW_JS = r"""
+    (value) => {
+    const el = document.querySelector("#graph");
+    if (!el) return null;
+    el.innerHTML = "";
+
+    if (!value || !value.nodes || value.nodes.length === 0) {
+        el.innerHTML = "<div class='graph-empty'>No graph</div>";
+        return null;
+    }
+
+    function ensureD3(cb) {
+        if (window.d3) return cb();
+        const s = document.createElement("script");
+        s.src = "https://cdn.jsdelivr.net/npm/d3@7";
+        s.onload = cb;
+        document.head.appendChild(s);
+    }
+
+    ensureD3(() => {
+        const width = el.clientWidth || 900;
+        const height = 600;
+
+        const svg = d3.select(el).append("svg")
+        .attr("viewBox", [0, 0, width, height])
+        .attr("preserveAspectRatio", "xMidYMid meet")
+        .style("width", "100%")
+        .style("height", "100%");
+
+        // Create zoomable container
+        const container = svg.append("g");
+
+        // Enable zoom & pan
+        svg.call(
+        d3.zoom()
+            .scaleExtent([0.2, 3])   // zoom limits (20%–300%)
+            .on("zoom", (event) => {
+            container.attr("transform", event.transform);
+            })
+        );
+
+        const sim = d3.forceSimulation(value.nodes)
+        .force("link", d3.forceLink(value.links).id(d => d.id).distance(140).strength(0.4))
+        .force("charge", d3.forceManyBody().strength(-220))
+        .force("center", d3.forceCenter(width / 2, height / 2));
+
+        const link = container.append("g")
+        .attr("stroke", "#999")
+        .attr("stroke-opacity", 0.6)
+        .selectAll("line")
+        .data(value.links)
+        .enter().append("line")
+        .attr("stroke-width", 1.5);
+
+        const edgeLabels = container.append("g")
+        .selectAll("text")
+        .data(value.links)
+        .enter().append("text")
+        .attr("font-size", 10)
+        .attr("fill", "#bbb")
+        .text(d => d.label);
+
+        const node = container.append("g")
+        .selectAll("circle")
+        .data(value.nodes)
+        .enter().append("circle")
+        .attr("r", 12)
+        .attr("fill", "#69b3a2")
+        .attr("stroke", "#2dd4bf")
+        .attr("stroke-width", 1.2)
+        .call(d3.drag()
+            .on("start", (event, d) => { if (!event.active) sim.alphaTarget(0.3).restart(); d.fx = d.x; d.fy = d.y; })
+            .on("drag", (event, d) => { d.fx = event.x; d.fy = event.y; })
+            .on("end",  (event, d) => { if (!event.active) sim.alphaTarget(0); d.fx = null; d.fy = null; })
+        );
+
+        const labels = container.append("g")
+        .selectAll("text")
+        .data(value.nodes)
+        .enter().append("text")
+        .attr("font-size", 12)
+        .attr("fill", "#ddd")
+        .attr("dy", 18)
+        .attr("text-anchor", "middle")
+        .text(d => d.id);
+
+        sim.on("tick", () => {
+        link
+            .attr("x1", d => d.source.x)
+            .attr("y1", d => d.source.y)
+            .attr("x2", d => d.target.x)
+            .attr("y2", d => d.target.y);
+
+        edgeLabels
+            .attr("x", d => (d.source.x + d.target.x) / 2)
+            .attr("y", d => (d.source.y + d.target.y) / 2);
+
+        node
+            .attr("cx", d => d.x)
+            .attr("cy", d => d.y);
+
+        labels
+            .attr("x", d => d.x)
+            .attr("y", d => d.y);
+        });
+    });
+
+    return null;
+    }
+    """
+
+
+    graph_data.change(lambda x: x, inputs=graph_data, outputs=graph_data).then(
+        None, inputs=graph_data, outputs=None, js=DRAW_JS
+    )
+
+if __name__ == "__main__":
+    demo.launch()
+    

utils.py

@@ -0,0 +1,24 @@
+from datetime import datetime, timezone
+import math
+
+def compute_freshness(ts, half_life_days=30):
+    """
+    Exponential decay score between 0 and 1.
+    - 1.0 -> now
+    - 0.5 -> half_life_days old
+    - Approaches 0 -> when docs get very old
+    """
+    if not ts:
+        return 0.5
+    if isinstance(ts, str):
+        ts = ts.replace("Z", "+00:00")
+        try:
+            ts = datetime.fromisoformat(ts)
+        except Exception:
+            return 0.5
+    if ts.tzinfo is None:
+        ts = ts.replace(tzinfo=timezone.utc)
+
+    age_days = (datetime.now(timezone.utc) - ts).total_seconds() / 86400.0
+    lam = math.log(2) / max(float(half_life_days), 1.0)
+    return math.exp(-lam * age_days)

vec.py

@@ -0,0 +1,76 @@
+import os
+import uuid
+from dotenv import load_dotenv
+from sentence_transformers import SentenceTransformer
+from qdrant_client import QdrantClient
+from qdrant_client.http import models
+
+import uuid
+from datetime import datetime, timezone
+
+load_dotenv()
+QDRANT_URL = os.getenv("QDRANT_URL")
+QDRANT_API_KEY = os.getenv("QDRANT_API_KEY")
+COLLECTION = "docs"
+
+model = SentenceTransformer("all-MiniLM-L6-v2") #fast embedder
+qdrant = QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)
+
+# Ensure collection exists (safe init .... we dont wipe on reload)
+def _ensure_collection():
+    try:
+        coll_info = qdrant.get_collection(collection_name=COLLECTION)
+        if not coll_info:
+            raise Exception("Collection not found")
+    except Exception:
+        print(f"⚠️ Collection '{COLLECTION}' not found. Creating fresh collection...")
+        qdrant.create_collection(
+            collection_name=COLLECTION,
+            vectors_config=models.VectorParams(
+                size=384,
+                distance=models.Distance.COSINE
+            ),
+        )
+
+_ensure_collection()
+
+def embed_and_upsert(chunks, source="user", timestamp=None):
+    if timestamp is None:
+        timestamp = datetime.now(timezone.utc).isoformat()
+
+    embeddings = model.encode(chunks).tolist()
+    points = []
+    for i, (chunk, emb) in enumerate(zip(chunks, embeddings)):
+        points.append(
+            models.PointStruct(
+                id=str(uuid.uuid4()),
+                vector=emb,
+                payload={
+                    "text": chunk,
+                    "source": source,
+                    "timestamp": timestamp,
+                    "chunk_id": i
+                }
+            )
+        )
+    qdrant.upsert(collection_name=COLLECTION, points=points, wait=True)
+    print(f"✅ Stored {len(points)} chunks in Qdrant (source={source}, ts={timestamp})")
+    return True
+
+def search(query: str, top_k: int = 5):
+    q_emb = model.encode([query])[0].tolist()
+    results = qdrant.search(
+        collection_name=COLLECTION,
+        query_vector=q_emb,
+        limit=top_k,
+        with_payload=True
+    )
+    return [
+        {
+            "chunk": r.payload.get("text", ""),
+            "cosine": r.score,
+            "timestamp": r.payload.get("timestamp"),
+            "source": r.payload.get("source")
+        }
+        for r in results
+    ]