Update app.py

app.py

@@ -1,12 +1,16 @@
-"""app.py — Gradio UI entry point (<200 lines, §11)."""
-import os, json, tempfile, time
+"""app.py — Gradio UI entry point.
+Tabs: Summary, UMAP, Pareto, Trial Log, Clusters, Top 3 Papers,
+      Methodology (3-LLM council + regex pipeline), Refinement Log,
+      Sheet 1-4, RQ Mismatch, Downloads.
+"""
+import os, json
 import pandas as pd, numpy as np
 import gradio as gr
 import plotly.express as px
 import plotly.graph_objects as go
-from agent import run_pipeline
+from agent import run_pipeline, METHODOLOGY_PATTERNS, TECHNIQUE_PATTERNS
 
-# ── CSV preview on upload ────────────────────────────────────────────────────
+# ── CSV preview ──────────────────────────────────────────────────────────────
 def _preview(file):
     if not file: return "Upload a Scopus CSV to begin."
     df = pd.read_csv(file.name)
@@ -19,53 +23,284 @@ def _preview(file):
     ok = "✅" if has_t and has_a and blanks_t < n and blanks_a < n else "❌"
     return (f"## {ok} CSV loaded — {n} entries\n\n"
         f"| Column | Present | Blank rows |\n|---|---|---|\n"
-        f"| title | {'✅' if has_t else '❌'} | {blanks_t} |\n"
+        f"| title  | {'✅' if has_t else '❌'} | {blanks_t} |\n"
         f"| abstract | {'✅' if has_a else '❌'} | {blanks_a} |\n\n"
-        f"**Usable papers:** {n - max(blanks_t,blanks_a)} / {n}")
+        f"**Usable papers:** {n - max(blanks_t, blanks_a)} / {n}")
+
+
+# ── Helper builders ──────────────────────────────────────────────────────────
+def _top_papers_df(top_papers: dict) -> pd.DataFrame:
+    rows = []
+    for cid in sorted(top_papers.keys()):
+        for p in top_papers[cid]:
+            rows.append({"Cluster": cid, "Label": p["cluster_label"],
+                         "Rank": p["rank"], "Title": p["title"],
+                         "Abstract Snippet": p["abstract_snippet"]})
+    return pd.DataFrame(rows)
+
+
+def _methodology_summary_df(methodology_data: dict, interps: dict) -> pd.DataFrame:
+    rows = []
+    for cid in sorted(methodology_data.keys()):
+        md    = methodology_data[cid]
+        label = interps.get(cid, {}).get("label", f"Cluster {cid}")
+        rows.append({
+            "Cluster":            cid,
+            "Label":              label,
+            "Dominant Method":    md.get("dominant_method", "—"),
+            "Dominant Technique": md.get("dominant_technique", "—"),
+            "Empirical %":        md.get("empirical_pct", 0),
+            "Theoretical %":      md.get("theoretical_pct", 0),
+            "Mixed %":            md.get("mixed_pct", 0),
+            "Methods (≥2 LLMs)":  ", ".join(
+                f"{m['name']} ({m['pct']}%, {m['agreement']})"
+                for m in md.get("methodologies", [])),
+            "Techniques (≥2 LLMs)": ", ".join(
+                f"{t['name']} ({t['pct']}%, {t['agreement']})"
+                for t in md.get("techniques", [])),
+            "Regex Confirmed":    ", ".join(md.get("regex_confirmed_consensus", [])) or "—",
+            "Regex Rejected":     ", ".join(md.get("regex_rejected_consensus", [])) or "—",
+        })
+    return pd.DataFrame(rows)
+
+
+def _extraction_pipeline_df(methodology_data: dict, interps: dict) -> pd.DataFrame:
+    """
+    One row per (cluster, method/technique) showing the full extraction trace:
+    which regex pattern fired, what text it matched, which LLMs confirmed it,
+    and whether it passed the ≥2-LLM gate.
+    """
+    rows = []
+    for cid in sorted(methodology_data.keys()):
+        md    = methodology_data[cid]
+        label = interps.get(cid, {}).get("label", f"Cluster {cid}")
+        scan  = md.get("regex_scan", {})
+
+        # Accepted items
+        for item in md.get("methodologies", []) + md.get("techniques", []):
+            name   = item["name"]
+            # Find regex hits for this category name
+            regex_hits = scan.get("methods", {}).get(name, []) or \
+                         scan.get("techniques", {}).get(name, [])
+            matched_text = ", ".join(
+                dict.fromkeys(h["match"] for h in regex_hits))[:80] if regex_hits else "—"
+            rows.append({
+                "Cluster":       cid,
+                "Label":         label,
+                "Item":          name,
+                "Type":          "Method" if item in md.get("methodologies",[]) else "Technique",
+                "Regex Match":   matched_text,
+                "Regex Fired":   "✅" if regex_hits else "❌",
+                "LLM Votes":     item["llm_votes"],
+                "Agreement":     item["agreement"],
+                "Avg Pct (%)":   item["pct"],
+                "Evidence":      item.get("evidence", "—"),
+                "Gate Passed":   "�� ACCEPTED",
+            })
+
+        # Rejected items (single LLM only)
+        for item in md.get("rejected_methods", []) + md.get("rejected_techniques", []):
+            name      = item["name"]
+            regex_hits = scan.get("methods", {}).get(name, []) or \
+                         scan.get("techniques", {}).get(name, [])
+            matched_text = ", ".join(
+                dict.fromkeys(h["match"] for h in regex_hits))[:80] if regex_hits else "—"
+            rows.append({
+                "Cluster":       cid,
+                "Label":         label,
+                "Item":          name,
+                "Type":          "Method" if item in md.get("rejected_methods",[]) else "Technique",
+                "Regex Match":   matched_text,
+                "Regex Fired":   "✅" if regex_hits else "❌",
+                "LLM Votes":     item["llm_votes"],
+                "Agreement":     item["agreement"],
+                "Avg Pct (%)":   item["pct"],
+                "Evidence":      item.get("evidence", "—"),
+                "Gate Passed":   "❌ REJECTED (single LLM)",
+            })
+
+    return pd.DataFrame(rows) if rows else pd.DataFrame()
+
+
+def _per_llm_methodology_df(methodology_data: dict, interps: dict) -> pd.DataFrame:
+    """Per-LLM raw methodology responses side-by-side."""
+    rows = []
+    for cid in sorted(methodology_data.keys()):
+        md    = methodology_data[cid]
+        label = interps.get(cid, {}).get("label", f"Cluster {cid}")
+        raw   = md.get("llm_raw", {})
+
+        def _fmt(r, key):
+            return " | ".join(
+                f"{i['name']} ({i.get('pct',0)}%)"
+                for i in r.get(key, [])
+            ) or "—"
+
+        rows.append({
+            "Cluster":             cid,
+            "Label":               label,
+            "Groq Methods":        _fmt(raw.get("groq",{}),    "methodologies"),
+            "Mistral Methods":     _fmt(raw.get("mistral",{}), "methodologies"),
+            "Gemini Methods":      _fmt(raw.get("gemini",{}),  "methodologies"),
+            "Groq Techniques":     _fmt(raw.get("groq",{}),    "techniques"),
+            "Mistral Techniques":  _fmt(raw.get("mistral",{}), "techniques"),
+            "Gemini Techniques":   _fmt(raw.get("gemini",{}),  "techniques"),
+            "Groq Emp/Theo/Mix":   f"{raw.get('groq',{}).get('empirical_pct',0)}/"
+                                   f"{raw.get('groq',{}).get('theoretical_pct',0)}/"
+                                   f"{raw.get('groq',{}).get('mixed_pct',0)}",
+            "Mistral Emp/Theo/Mix":f"{raw.get('mistral',{}).get('empirical_pct',0)}/"
+                                   f"{raw.get('mistral',{}).get('theoretical_pct',0)}/"
+                                   f"{raw.get('mistral',{}).get('mixed_pct',0)}",
+            "Gemini Emp/Theo/Mix": f"{raw.get('gemini',{}).get('empirical_pct',0)}/"
+                                   f"{raw.get('gemini',{}).get('theoretical_pct',0)}/"
+                                   f"{raw.get('gemini',{}).get('mixed_pct',0)}",
+        })
+    return pd.DataFrame(rows)
+
+
+def _regex_hits_df(methodology_data: dict, interps: dict) -> pd.DataFrame:
+    """
+    One row per (cluster, pattern, matched text) so the user can see exactly
+    which regex fired on which word in which paper.
+    """
+    rows = []
+    for cid in sorted(methodology_data.keys()):
+        md    = methodology_data[cid]
+        label = interps.get(cid, {}).get("label", f"Cluster {cid}")
+        scan  = md.get("regex_scan", {})
+
+        for category, hits in scan.get("methods", {}).items():
+            for h in hits:
+                rows.append({"Cluster": cid, "Label": label,
+                             "Bank": "Methodology", "Pattern Category": category,
+                             "Matched Text": h["match"], "Paper #": h["doc"],
+                             "Char Span": f"{h['span'][0]}–{h['span'][1]}"})
+
+        for category, hits in scan.get("techniques", {}).items():
+            for h in hits:
+                rows.append({"Cluster": cid, "Label": label,
+                             "Bank": "Technique", "Pattern Category": category,
+                             "Matched Text": h["match"], "Paper #": h["doc"],
+                             "Char Span": f"{h['span'][0]}–{h['span'][1]}"})
+
+    return pd.DataFrame(rows) if rows else pd.DataFrame()
+
+
+def _methodology_bar_chart(methodology_data: dict, interps: dict) -> go.Figure:
+    labels_list, empirical, theoretical, mixed = [], [], [], []
+    for cid in sorted(methodology_data.keys()):
+        md = methodology_data[cid]
+        labels_list.append(interps.get(cid,{}).get("label", f"C{cid}")[:30])
+        empirical.append(md.get("empirical_pct", 0))
+        theoretical.append(md.get("theoretical_pct", 0))
+        mixed.append(md.get("mixed_pct", 0))
+
+    fig = go.Figure()
+    fig.add_trace(go.Bar(name="Empirical %",   x=labels_list, y=empirical,   marker_color="#3dba7a"))
+    fig.add_trace(go.Bar(name="Theoretical %", x=labels_list, y=theoretical, marker_color="#5b9cf6"))
+    fig.add_trace(go.Bar(name="Mixed %",       x=labels_list, y=mixed,       marker_color="#f5a623"))
+    fig.update_layout(
+        barmode="stack", template="plotly_dark", height=420,
+        paper_bgcolor="#0d1117", plot_bgcolor="#161b22",
+        title="Research Orientation per Cluster — Averaged across Groq + Mistral + Gemini",
+        xaxis_title="Cluster", yaxis_title="Percentage (%)",
+        font=dict(size=11), legend=dict(orientation="h", y=1.12),
+        xaxis_tickangle=-35,
+    )
+    return fig
+
+
+def _regex_pattern_info() -> str:
+    m_list = "\n".join(f"- **{k}**: `{v.pattern}`" for k,v in METHODOLOGY_PATTERNS.items())
+    t_list = "\n".join(f"- **{k}**: `{v.pattern}`" for k,v in TECHNIQUE_PATTERNS.items())
+    return (
+        "### How Methodology Extraction Works\n\n"
+        "**Step 1 — Regex Pre-Scan**  \n"
+        "Two compiled pattern banks (case-insensitive) are run against each representative abstract. "
+        "Every match is recorded with its exact character span, matched text, and paper number. "
+        "This produces ground-truth hints that are injected into the LLM prompt.\n\n"
+        "**Step 2 — 3-LLM Council**  \n"
+        "Groq (llama-3.1-8b), Mistral (mistral-small), and Gemini (gemini-2.5-flash) each receive "
+        "the same prompt: the regex evidence + the full abstracts. Each LLM must confirm or reject "
+        "the regex hits and may add methods/techniques it finds in the text. "
+        "Each LLM also provides an evidence quote (≤15 words) for every item it names.\n\n"
+        "**Step 3 — Consolidation (≥2-LLM gate)**  \n"
+        "A method or technique only survives if at least 2 out of 3 LLMs named it. "
+        "Percentages are averaged across agreeing LLMs. Items named by only one LLM are marked "
+        "REJECTED and shown in the extraction pipeline table.\n\n"
+        "**Step 4 — Orientation Percentages**  \n"
+        "Empirical / Theoretical / Mixed percentages are averaged across all 3 LLMs and shown "
+        "in the stacked bar chart above.\n\n"
+        "---\n\n"
+        "#### Methodology Pattern Bank\n" + m_list +
+        "\n\n#### Technique Pattern Bank\n" + t_list
+    )
+
+
+def _refinement_df(refinement_log: list) -> pd.DataFrame:
+    if not refinement_log:
+        return pd.DataFrame(columns=["Cluster","Iteration","Old Label","New Label",
+                                     "Issues","Improvement","Hallucination Detected"])
+    return pd.DataFrame([{
+        "Cluster":               r["cluster"],
+        "Iteration":             r["iteration"],
+        "Old Label":             r["old_label"],
+        "New Label":             r["new_label"],
+        "Issues":                "; ".join(r.get("issues",[])),
+        "Improvement":           r["improvement_score"],
+        "Hallucination Detected":r["hallucination_detected"],
+    } for r in refinement_log])
+
 
 # ── Pipeline runner ──────────────────────────────────────────────────────────
-def _run(file, gk, mk, gek, n_trials, progress=gr.Progress(track_tqdm=True)):
+def _run(file, gk, mk, gek, n_trials, n_optimize,
+         progress=gr.Progress(track_tqdm=True)):
     if not file: raise gr.Error("Upload a CSV first.")
-    gk = gk.strip() or os.getenv("GROQ_API_KEY","")
-    mk = mk.strip() or os.getenv("MISTRAL_API_KEY","")
+    gk  = gk.strip()  or os.getenv("GROQ_API_KEY","")
+    mk  = mk.strip()  or os.getenv("MISTRAL_API_KEY","")
     gek = gek.strip() or os.getenv("GEMINI_API_KEY","")
     if not all([gk,mk,gek]): raise gr.Error("All 3 API keys required.")
+
     progress(0.05, desc="📥 Loading CSV…")
-    progress(0.1, desc="🔬 Embedding with SPECTER-2 (this takes a few minutes)…")
-    r = run_pipeline(file.name, gk, mk, gek, int(n_trials))
+    progress(0.10, desc="🔬 Embedding with SPECTER-2 (this takes a few minutes)…")
+    r = run_pipeline(file.name, gk, mk, gek, int(n_trials), int(n_optimize))
     if r.get("error"): raise gr.Error(r["error"])
-    progress(0.95, desc="📊 Building outputs…")
-    td, interps = r["topic_data"], r.get("interpretations",{})
-    disc, met = td["discipline"], td["metrics"]
-    ar = r.get("agreement_rates",{})
-    # ── Summary metrics (styled like reference) ──
+
+    progress(0.85, desc="📊 Building outputs…")
+    td, interps = r["topic_data"], r.get("interpretations", {})
+    disc, met   = td["discipline"], td["metrics"]
+    ar          = r.get("agreement_rates", {})
+    rl          = r.get("refinement_log", [])
+
     def _s(ok): return "✅ PASS" if ok else "❌ FAIL"
-    summary = (f"## Pipeline Complete — {disc['n_clusters']} clusters discovered\n\n"
+    summary = (
+        f"## Pipeline Complete — {disc['n_clusters']} clusters discovered\n\n"
         f"| Criterion | Value | Status |\n|---|---|---|\n"
         f"| Max cluster mass | {round(disc['max_mass_pct']*100,1)}% | {_s(disc['max_mass_ok'])} |\n"
         f"| Min cluster size | {disc['min_size']} | {_s(disc['min_size_ok'])} |\n"
         f"| Persistence (mean) | {round(met['persistence'],4)} | — |\n"
         f"| DBCV | {round(met['dbcv'],4)} | — |\n"
-        f"| Stability ({3} seeds) | {round(met['stability'],4)} | — |\n\n"
+        f"| Stability (3 seeds) | {round(met['stability'],4)} | — |\n\n"
         f"**Trials:** {td['n_trials_run']} (best #{td['best_trial']}) · "
-        f"**Agreement:** Triple {ar.get('triple',0)}% · Two+ {ar.get('two_or_more',0)}%")
-    # ── UMAP scatter ──
+        f"**Agreement:** Triple {ar.get('triple',0)}% · Two+ {ar.get('two_or_more',0)}% · "
+        f"**Optimization passes:** {n_optimize} · **Labels refined:** {len(rl)}"
+    )
+
     u2d = np.array(td["umap_2d"])
     sdf = pd.DataFrame({"UMAP-1":u2d[:,0],"UMAP-2":u2d[:,1],
         "Cluster":[str(l) for l in td["labels"]],
         "Doc":[d[:60] for d in td["documents"]]})
     fig = px.scatter(sdf, x="UMAP-1", y="UMAP-2", color="Cluster",
         hover_data=["Doc"], opacity=0.75,
-        title=f"2-D UMAP visualisation of SPECTER-2 embeddings")
+        title="2-D UMAP visualisation of SPECTER-2 embeddings")
     fig.update_layout(template="plotly_dark", height=500,
-        paper_bgcolor="#0d1117", plot_bgcolor="#161b22",
-        font=dict(size=11))
-    # ── Trial log ──
+        paper_bgcolor="#0d1117", plot_bgcolor="#161b22", font=dict(size=11))
+
     tl = pd.DataFrame(td["trial_log"])
     tl_cols = [c for c in ["trial","discipline_pass","n_clusters","persistence",
         "dbcv","max_mass_pct","min_size","n_noise"] if c in tl.columns]
     tl_show = tl[tl_cols] if not tl.empty else pd.DataFrame()
-    # ── Pareto front ──
+
     pfig = go.Figure()
     if not tl.empty:
         for passed, color, name in [(True,"#3dba7a","PASS"),(False,"#e04d4d","FAIL")]:
@@ -74,73 +309,165 @@ def _run(file, gk, mk, gek, n_trials, progress=gr.Progress(track_tqdm=True)):
                 pfig.add_trace(go.Scatter(x=sub["max_mass_pct"],y=sub["persistence"],
                     mode="markers",marker=dict(size=8,color=color),name=name,
                     text=sub["trial"],hovertemplate="Trial %{text}<br>Mass: %{x:.0%}<br>Pers: %{y:.3f}"))
-        pfig.add_vline(x=0.25, line_dash="dash", line_color="#5a6480",
-            annotation_text="25% rule")
-    pfig.update_layout(template="plotly_dark", height=400,
-        paper_bgcolor="#0d1117", plot_bgcolor="#161b22",
+        pfig.add_vline(x=0.25,line_dash="dash",line_color="#5a6480",annotation_text="25% rule")
+    pfig.update_layout(template="plotly_dark",height=400,
+        paper_bgcolor="#0d1117",plot_bgcolor="#161b22",
         title="Pareto front — Persistence vs Max cluster mass",
-        xaxis_title="Max cluster mass (lower is better)",
-        yaxis_title="Persistence (higher is better)", font=dict(size=11))
-    # ── Cluster table ──
-    rows = []
+        xaxis_title="Max cluster mass",yaxis_title="Persistence",font=dict(size=11))
+
+    cdf_rows = []
     for cid in sorted(interps.keys()):
         v = interps[cid]
-        rows.append({"Cluster":cid,"Label":v["label"],"Agreement":v["agreement"],
+        cdf_rows.append({"Cluster":cid,"Label":v["label"],"Agreement":v["agreement"],
             "Strong":v["strong"],"Weak":v["weak"],
             "Persistence":round(v.get("persistence",0),4),
             "Keyphrases":", ".join(v.get("keyphrases",[]))})
-    cdf = pd.DataFrame(rows)
-    # ── 4 separate sheets ──
+    cdf = pd.DataFrame(cdf_rows)
+
     sheets = r.get("sheets",{})
     s1 = pd.DataFrame(sheets.get(1,[])); s2 = pd.DataFrame(sheets.get(2,[]))
     s3 = pd.DataFrame(sheets.get(3,[])); s4 = pd.DataFrame(sheets.get(4,[]))
     sp = r.get("sheet_paths",{})
     mdf = pd.DataFrame(r.get("mismatch_table",[]))
+
+    md_data = r.get("methodology_data", {})
+
+    top_papers_df       = _top_papers_df(r.get("top_papers", {}))
+    method_summary_df   = _methodology_summary_df(md_data, interps)
+    method_chart        = _methodology_bar_chart(md_data, interps)
+    extraction_df       = _extraction_pipeline_df(md_data, interps)
+    per_llm_df          = _per_llm_methodology_df(md_data, interps)
+    regex_hits_df       = _regex_hits_df(md_data, interps)
+    pattern_info        = _regex_pattern_info()
+    refine_df           = _refinement_df(rl)
+
     progress(1.0, desc="✅ Done!")
-    dl_files = [f for f in
-        [sp.get(1), sp.get(2), sp.get(3), sp.get(4), r.get("json_path")]
-        if f is not None]
-    return (summary, fig, pfig, tl_show, cdf, s1, s2, s3, s4,
-            dl_files if dl_files else None, mdf)
+    dl_files = [f for f in [sp.get(1),sp.get(2),sp.get(3),sp.get(4),r.get("json_path")] if f]
+
+    return (summary, fig, pfig, tl_show, cdf,
+            top_papers_df,
+            method_chart, method_summary_df, extraction_df, per_llm_df,
+            regex_hits_df, pattern_info,
+            refine_df,
+            s1, s2, s3, s4,
+            dl_files if dl_files else None,
+            mdf)
+
 
 # ── UI ─────────────────────��─────────────────────────────────────────────────
 css = ".gradio-container{background:#0d1117!important;color:#c9d1d9!important}" \
       "footer{display:none!important}"
-with gr.Blocks(theme=gr.themes.Base(primary_hue="blue",neutral_hue="slate"),
+
+with gr.Blocks(theme=gr.themes.Base(primary_hue="blue", neutral_hue="slate"),
                css=css, title="SPECTER-2 Topic Analyzer") as demo:
     gr.Markdown("# 📐 SPECTER-2 Topic Analyzer")
+
     with gr.Row():
         with gr.Column(scale=1):
-            file_in = gr.File(label="Upload Scopus CSV", file_types=[".csv"])
+            file_in    = gr.File(label="Upload Scopus CSV", file_types=[".csv"])
             preview_out = gr.Markdown("Upload a CSV to see stats.")
-            groq_in = gr.Textbox(label="Groq API Key", type="password",
-                placeholder="or set GROQ_API_KEY env var")
+            groq_in    = gr.Textbox(label="Groq API Key", type="password",
+                            placeholder="or set GROQ_API_KEY env var")
             mistral_in = gr.Textbox(label="Mistral API Key", type="password",
-                placeholder="or set MISTRAL_API_KEY env var")
-            gemini_in = gr.Textbox(label="Gemini API Key", type="password",
-                placeholder="or set GEMINI_API_KEY env var")
-            trials_in = gr.Slider(10,100,50,step=5,label="Optuna Trials")
-            run_btn = gr.Button("▶ Run Full Pipeline", variant="primary", size="lg")
+                            placeholder="or set MISTRAL_API_KEY env var")
+            gemini_in  = gr.Textbox(label="Gemini API Key", type="password",
+                            placeholder="or set GEMINI_API_KEY env var")
+            trials_in  = gr.Slider(10, 100, 50, step=5, label="Optuna Trials")
+            optimize_in = gr.Slider(1, 5, 1, step=1,
+                            label="🔁 Optimization Passes",
+                            info="Each pass: LLM critic audits labels for hallucinations. "
+                                 "1 = disabled. 2–5 = progressive refinement.")
+            run_btn    = gr.Button("▶ Run Full Pipeline", variant="primary", size="lg")
+
         with gr.Column(scale=3):
             with gr.Tabs():
-                with gr.Tab("Summary"): summary_out = gr.Markdown()
-                with gr.Tab("2-D UMAP"): scatter_out = gr.Plot()
-                with gr.Tab("Pareto Front"): pareto_out = gr.Plot()
-                with gr.Tab("Trial Log"): trial_out = gr.Dataframe()
-                with gr.Tab("Clusters"): cluster_out = gr.Dataframe()
-                with gr.Tab("Sheet 1 — Groq"): s1_out = gr.Dataframe()
+
+                with gr.Tab("Summary"):
+                    summary_out = gr.Markdown()
+
+                with gr.Tab("2-D UMAP"):
+                    scatter_out = gr.Plot()
+
+                with gr.Tab("Pareto Front"):
+                    pareto_out = gr.Plot()
+
+                with gr.Tab("Trial Log"):
+                    trial_out = gr.Dataframe()
+
+                with gr.Tab("Clusters"):
+                    cluster_out = gr.Dataframe()
+
+                with gr.Tab("🗞 Top 3 Papers"):
+                    gr.Markdown("### Top 3 Representative Papers per Cluster\n"
+                                "Ranked by cosine similarity to the cluster centroid "
+                                "in SPECTER-2 embedding space.")
+                    top_papers_out = gr.Dataframe(
+                        headers=["Cluster","Label","Rank","Title","Abstract Snippet"],
+                        wrap=True)
+
+                with gr.Tab("🔬 Methodology — Summary"):
+                    gr.Markdown("### Consolidated Methodology Results\n"
+                                "Only items agreed by **≥ 2 out of 3 LLMs** (Groq + Mistral + Gemini) "
+                                "appear here. Percentages averaged across agreeing LLMs.")
+                    method_chart_out  = gr.Plot()
+                    method_summary_out = gr.Dataframe(wrap=True)
+
+                with gr.Tab("⚙ Methodology — Extraction Pipeline"):
+                    gr.Markdown("### Full Extraction Trace\n"
+                                "One row per method/technique showing: which regex pattern fired, "
+                                "the exact matched text, how many LLMs agreed, and whether it "
+                                "passed the ≥2-LLM gate.")
+                    extraction_out = gr.Dataframe(wrap=True)
+
+                with gr.Tab("🤖 Methodology — Per-LLM Votes"):
+                    gr.Markdown("### Raw Per-LLM Methodology Responses\n"
+                                "Side-by-side view of what each LLM independently extracted "
+                                "before consolidation.")
+                    per_llm_out = gr.Dataframe(wrap=True)
+
+                with gr.Tab("🔍 Regex Hits"):
+                    gr.Markdown("### Regex Pattern Matches\n"
+                                "Every regex match with its exact character span, matched text, "
+                                "and which paper (1–3) it came from. This is the ground-truth "
+                                "evidence fed to all 3 LLMs.")
+                    regex_hits_out  = gr.Dataframe(wrap=True)
+                    regex_info_out  = gr.Markdown()
+
+                with gr.Tab("🔁 Refinement Log"):
+                    gr.Markdown("### Optimization Refinement Log\n"
+                                "Changes made by the Groq critic per optimization pass. "
+                                "A label is only changed when improvement_score > 0.15 "
+                                "OR hallucination was detected, AND the new label passes "
+                                "the keyphrase grounding check.")
+                    refine_out = gr.Dataframe(
+                        headers=["Cluster","Iteration","Old Label","New Label",
+                                 "Issues","Improvement","Hallucination Detected"],
+                        wrap=True)
+
+                with gr.Tab("Sheet 1 — Groq"):    s1_out = gr.Dataframe()
                 with gr.Tab("Sheet 2 — Mistral"): s2_out = gr.Dataframe()
-                with gr.Tab("Sheet 3 — Gemini"): s3_out = gr.Dataframe()
+                with gr.Tab("Sheet 3 — Gemini"):  s3_out = gr.Dataframe()
                 with gr.Tab("Sheet 4 — Consolidated"): s4_out = gr.Dataframe()
-                with gr.Tab("RQ Mismatch"): mismatch_out = gr.Dataframe()
+                with gr.Tab("RQ Mismatch"):       mismatch_out = gr.Dataframe()
                 with gr.Tab("Downloads"):
                     dl_out = gr.File(label="All sheet CSVs + topics.json",
                                      file_count="multiple")
+
     file_in.change(_preview, inputs=[file_in], outputs=[preview_out])
-    run_btn.click(_run,
-        inputs=[file_in, groq_in, mistral_in, gemini_in, trials_in],
-        outputs=[summary_out, scatter_out, pareto_out, trial_out, cluster_out,
-                 s1_out, s2_out, s3_out, s4_out, dl_out, mismatch_out])
+
+    run_btn.click(
+        _run,
+        inputs=[file_in, groq_in, mistral_in, gemini_in, trials_in, optimize_in],
+        outputs=[
+            summary_out, scatter_out, pareto_out, trial_out, cluster_out,
+            top_papers_out,
+            method_chart_out, method_summary_out, extraction_out, per_llm_out,
+            regex_hits_out, regex_info_out,
+            refine_out,
+            s1_out, s2_out, s3_out, s4_out,
+            dl_out, mismatch_out,
+        ],
+    )
 
 if __name__ == "__main__":
-    demo.launch(server_name="0.0.0.0", server_port=7860)
+    demo.launch(server_name="0.0.0.0", server_port=7860)