Create app.py

app.py

@@ -0,0 +1,219 @@
+import gradio as gr
+import json, re, math, os
+from collections import Counter, defaultdict
+
+
+# ===============================================================
+# UTILITIES
+# ===============================================================
+
+def tokenize(text):
+    return re.findall(r"[A-Za-z0-9']+", text.lower())
+
+def text_vector(text):
+    return Counter(tokenize(text))
+
+def centroid(docs):
+    C = Counter()
+    for d in docs:
+        C.update(text_vector(d["text"]))
+    return C
+
+def cosine(a, b):
+    num = 0
+    da = 0
+    db = 0
+    for k in set(a.keys()) | set(b.keys()):
+        va = a.get(k,0)
+        vb = b.get(k,0)
+        num += va*vb
+        da += va*va
+        db += vb*vb
+    if da == 0 or db == 0:
+        return 0
+    return num / math.sqrt(da*db)
+
+
+# ===============================================================
+# MAIN LOAD FUNCTION
+# ===============================================================
+
+def load_jsonl(jsonl_file):
+    if jsonl_file is None:
+        return None, None, "⚠ No file uploaded."
+
+    records = []
+    with open(jsonl_file.name, "r", encoding="utf8") as f:
+        for line in f:
+            try:
+                records.append(json.loads(line))
+            except:
+                pass
+
+    # build clusters
+    cluster_map = defaultdict(list)
+    for r in records:
+        cluster_map[r.get("cluster", -1)].append(r)
+
+    # build BM25
+    docs_text = [r["text"] for r in records]
+    tokenized_docs = [tokenize(t) for t in docs_text]
+
+    doc_freq = Counter()
+    for toks in tokenized_docs:
+        for t in set(toks):
+            doc_freq[t] += 1
+
+    Ndocs = len(records)
+    avg_len = sum(len(t) for t in tokenized_docs) / Ndocs
+
+    # Precompute centroids
+    centroids = {cid: centroid(docs) for cid, docs in cluster_map.items()}
+
+    return {
+        "records": records,
+        "cluster_map": cluster_map,
+        "tokenized_docs": tokenized_docs,
+        "doc_freq": doc_freq,
+        "Ndocs": Ndocs,
+        "avg_len": avg_len,
+        "centroids": centroids,
+    }, sorted(cluster_map.keys()), f"Loaded {len(records)} records."
+
+
+# ===============================================================
+# BM25 SEARCH
+# ===============================================================
+
+def bm25_score(query, doc_toks, doc_freq, Ndocs, avg_len):
+    k=1.5; b=0.75
+    score = 0
+    q_toks = tokenize(query)
+    for q in q_toks:
+        df = doc_freq.get(q,0)
+        if df == 0:
+            continue
+        idf = math.log((Ndocs - df + 0.5)/(df + 0.5) + 1)
+        tf = doc_toks.count(q)
+        denom = tf + k*(1 - b + b*(len(doc_toks)/avg_len))
+        score += idf*(tf*(k+1))/denom
+    return score
+
+
+# ===============================================================
+# GRADIO INTERFACE FUNCTIONS
+# ===============================================================
+
+def do_view_cluster(state, cid):
+    if state is None:
+        return "⚠ Upload a file first."
+    try:
+        cid = int(cid)
+    except:
+        return "Enter a valid number."
+
+    cluster_map = state["cluster_map"]
+
+    if cid not in cluster_map:
+        return "❌ Cluster not found."
+
+    out = [f"=== Cluster {cid} ({len(cluster_map[cid])} docs) ===\n"]
+    for d in cluster_map[cid][:20]:
+        t = d["text"].strip()
+        if len(t) > 1200:
+            t = t[:1200] + " … [truncated]"
+        out.append(f"\n--- id={d.get('id')} ---\n{t}\n")
+    return "\n".join(out)
+
+
+def do_search(state, query):
+    if state is None:
+        return "⚠ Upload a file first."
+
+    records = state["records"]
+    tokenized_docs = state["tokenized_docs"]
+    doc_freq = state["doc_freq"]
+    Ndocs = state["Ndocs"]
+    avg_len = state["avg_len"]
+
+    scores = []
+    for r, toks in zip(records, tokenized_docs):
+        s = bm25_score(query, toks, doc_freq, Ndocs, avg_len)
+        if s > 0:
+            scores.append((s, r))
+    scores.sort(reverse=True, key=lambda x: x[0])
+
+    out = [f"=== Results for '{query}' ==="]
+    for s, r in scores[:15]:
+        out.append(f"\nScore {s:.2f} — Cluster {r['cluster']} — id={r['id']}\n{r['text'][:500]}…\n")
+
+    return "\n".join(out)
+
+
+def do_show_topics(state):
+    if state is None:
+        return "⚠ Upload a file first."
+
+    STOPWORDS = set("""
+the and to of a in is this that for on with as be or by from at
+an it are was you your if but have we they his her she their our
+subject re fw message thereof all may any doc email
+""".split())
+
+    out = ["=== Cluster Topics ==="]
+    for cid, cent in state["centroids"].items():
+        filtered = {w:c for w,c in cent.items()
+                    if w not in STOPWORDS and len(w) > 2 and c > 1}
+        top = [w for w,_ in Counter(filtered).most_common(6)]
+        out.append(f"Cluster {cid:<4} | {' '.join(top)}")
+
+    return "\n".join(out)
+
+
+def do_entity_search(state, name):
+    if state is None:
+        return "⚠ Upload a file first."
+
+    hits = []
+    cluster_map = state["cluster_map"]
+
+    for cid, docs in cluster_map.items():
+        count = sum(name.lower() in d["text"].lower() for d in docs)
+        if count:
+            hits.append((count, cid))
+
+    hits.sort(reverse=True)
+
+    out = [f"=== Clusters mentioning '{name}' ==="]
+    for count, cid in hits[:20]:
+        out.append(f"Cluster {cid}: {count} hits")
+    return "\n".join(out)
+
+
+# ===============================================================
+# GRADIO UI
+# ===============================================================
+
+with gr.Blocks(title="Epstein Semantic Explorer v5") as demo:
+    gr.Markdown("# **Epstein Semantic Explorer v5**  
+Upload your `epstein_semantic.jsonl` file.")
+
+    with gr.Row():
+        jsonl_file = gr.File(label="Upload JSONL dataset")
+        load_btn = gr.Button("Load Dataset")
+
+    state = gr.State()
+    clusters_box = gr.Number(label="Cluster # to View", value=96)
+    query_box = gr.Textbox(label="Search Keyword")
+    entity_box = gr.Textbox(label="Search for Entity (name)")
+
+    output = gr.Textbox(label="Output", lines=30)
+
+    load_btn.click(load_jsonl, inputs=[jsonl_file], outputs=[state, clusters_box, output])
+    clusters_box.change(do_view_cluster, inputs=[state, clusters_box], outputs=output)
+    query_box.submit(do_search, inputs=[state, query_box], outputs=output)
+    entity_box.submit(do_entity_search, inputs=[state, entity_box], outputs=output)
+
+    gr.Button("Show Topics").click(do_show_topics, inputs=[state], outputs=output)
+
+demo.launch()