Delete app.py

app.py

@@ -1,413 +0,0 @@
-"""app.py — Compact Gradio + Mistral + BERTopic. Version 2.1.0 | 4 April 2026. ZERO for/while/if.
-
-FUNCTIONS IN THIS FILE:
-  _latest_output()  — Scans /tmp for newest rq4_* file → feeds download button
-  respond()         — Core chat handler: takes message + history + file → yields agent response
-  gr.Blocks()       — One-page UI: header + upload + chatbot + input + download
-"""
-import os
-import glob
-import gradio as gr
-from langchain_mistralai import ChatMistralAI
-from langgraph.prebuilt import create_react_agent
-from langgraph.checkpoint.memory import MemorySaver
-from agent import SYSTEM_PROMPT, get_local_tools
-
-print(">>> app.py: imports complete")
-
-# ═══════════════════════════════════════════════
-# AGENT SETUP — Mistral brain + 5 BERTopic tools
-# ═══════════════════════════════════════════════
-llm = ChatMistralAI(model="mistral-small-latest", temperature=0, timeout=300)
-tools = get_local_tools()
-agent = create_react_agent(model=llm, tools=tools, prompt=SYSTEM_PROMPT, checkpointer=MemorySaver())
-print(f">>> app.py: agent ready ({len(tools)} tools, Mistral Large)")
-
-_msg_count = 0
-_uploaded = {"path": ""}
-
-# ═══════════════════════════════════════════════
-# COMPACT HEADER — fits in ~60px
-# ═══════════════════════════════════════════════
-HEADER_HTML = """
-<style>
-@import url('https://fonts.googleapis.com/css2?family=DM+Sans:wght@400;500;600;700&display=swap');
-.gradio-container {font-family: 'DM Sans', sans-serif !important; padding: 4px 8px !important; max-width: 100% !important;}
-footer {display: none !important;}
-.section-box {border: 1px solid #e2e8f0; border-radius: 8px; padding: 8px 12px; margin-bottom: 6px; background: #fafbfc;}
-.section-label {font-size: 0.75em; font-weight: 600; color: #64748b; text-transform: uppercase; letter-spacing: 0.5px; margin-bottom: 4px;}
-</style>
-<div style="background: linear-gradient(135deg, #0f172a 0%, #1e293b 50%, #334155 100%); border-radius: 8px; padding: 8px 16px; margin-bottom: 4px; color: white; display: flex; align-items: center; gap: 10px;">
-    <span style="font-size: 1.3em;">🔬</span>
-    <div>
-        <span style="font-size: 1em; font-weight: 700; color: #e0e0ff;">Topic Modelling — Agentic AI</span>
-        <span style="font-size: 0.65em; color: #94a3b8; margin-left: 8px;">Mistral 🇫🇷 · Cosine Clustering · 384d · B&C Thematic Analysis</span>
-    </div>
-</div>
-"""
-
-# ═══════════════════════════════════════════════
-# FUNCTION 1: Find latest output for download
-# ═══════════════════════════════════════════════
-def _latest_output():
-    """Scan /tmp for ALL rq4_* files, sorted by phase order.
-    Returns list of filepaths for gr.File download component."""
-    # Phase order: summaries → labels → themes → taxonomy → charts → comparison → narrative
-    phase_order = {"summaries": 1, "labels": 2, "themes": 3, "taxonomy": 4,
-                   "emb": 0, "intertopic": 5, "bars": 6, "hierarchy": 7,
-                   "heatmap": 8, "comparison": 9, "narrative": 10}
-    files = (glob.glob("/tmp/rq4_*.csv") + glob.glob("/tmp/rq4_*.html")
-             + glob.glob("/tmp/rq4_*.txt") + glob.glob("/tmp/checkpoints/rq4_*.json"))
-    # Sort by phase order using filename keywords
-    scored = list(map(
-        lambda f: (sum(v * (k in f) for k, v in phase_order.items()), f), files))
-    scored.sort(key=lambda x: x[0])
-    return list(map(lambda x: x[1], scored)) or None
-
-
-
-
-def _build_progress():
-    """Build HTML progress pipeline showing which B&C phases are complete."""
-    checks = [
-        ("① Load", bool(glob.glob("/tmp/checkpoints/rq4_*_summaries.json") or glob.glob("/tmp/checkpoints/rq4_*_emb.npy"))),
-        ("② Codes", bool(glob.glob("/tmp/checkpoints/rq4_*_labels.json"))),
-        ("③ Themes", bool(glob.glob("/tmp/checkpoints/rq4_*_themes.json"))),
-        ("④ Review", bool(glob.glob("/tmp/checkpoints/rq4_*_themes.json"))),
-        ("⑤ Names", bool(glob.glob("/tmp/checkpoints/rq4_*_themes.json"))),
-        ("⑤½ PAJAIS", bool(glob.glob("/tmp/checkpoints/rq4_*_taxonomy_map.json"))),
-        ("⑥ Report", bool(glob.glob("/tmp/rq4_comparison.csv") or glob.glob("/tmp/rq4_narrative.txt"))),
-    ]
-    done_bg = "#22c55e"
-    todo_bg = "#e2e8f0"
-    done_fg = "white"
-    todo_fg = "#64748b"
-    items = " → ".join(list(map(
-        lambda c: (
-            "<span style='padding:3px 8px;border-radius:4px;"
-            + "background:" + (done_bg * c[1] or todo_bg) + ";"
-            + "color:" + (done_fg * c[1] or todo_fg) + ";"
-            + "font-weight:600;font-size:0.8em;'>"
-            + c[0] + " " + ("✅" * c[1] or "⬜") + "</span>"
-        ), checks)))
-    return "<div style='padding:6px 0;text-align:center;'>" + items + "</div>"
-
-
-# ═══════════════════════════════════════════════
-# FUNCTION 2: Chat handler — core of the app
-# ═══════════════════════════════════════════════
-def respond(message, chat_history, uploaded_file):
-    """Handle one chat turn:
-    1. Store uploaded file path (if new upload)
-    2. Append file context to message so agent knows where CSV is
-    3. Show progress bubble immediately (user sees instant feedback)
-    4. Invoke agent (Mistral brain decides which tools to call)
-    5. Replace progress bubble with agent's actual response
-    6. Update download link to latest output file
-
-    Uses single thread_id="session" so agent remembers across turns.
-    Agent asks clarification FIRST (via SYSTEM_PROMPT) before running heavy tools."""
-    global _msg_count
-    _msg_count += 1
-
-    # Store file path — no if/else, uses `or` short-circuit
-    _uploaded["path"] = uploaded_file or _uploaded.get("path", "")
-
-    # Guard: tell agent when no file uploaded (prevents hallucinated filepath)
-    file_note = (f"\n[CSV file at: {_uploaded['path']}]" * bool(_uploaded["path"])
-                ) or "\n[No CSV uploaded yet — ask user to upload a file first]"
-
-    # Context: tell agent what phase we're in based on checkpoint files
-    phase_context = (
-        "\n[Phase context: labels exist]" * bool(glob.glob("/tmp/checkpoints/rq4_*_labels.json"))
-        or "\n[Phase context: embeddings exist]" * bool(glob.glob("/tmp/checkpoints/rq4_*_emb.npy"))
-        or "\n[Phase context: fresh start]"
-    )
-
-    text = ((message or "").strip() or "Analyze my Scopus CSV") + file_note + phase_context
-    print(f"\n{'='*60}\n>>> MSG #{_msg_count}: '{text[:120]}'\n{'='*60}")
-
-    # Yield progress bubble immediately — user sees instant response
-    chat_history = chat_history + [
-        {"role": "user", "content": (message or "").strip()},
-        {"role": "assistant", "content": "🔬 **Working...**  _Agent is thinking..._"},
-    ]
-    yield chat_history, "", _latest_output()
-
-    # Shared session — agent remembers across messages (loaded CSV, phase, etc.)
-    # handle_tool_error=True on all tools prevents session poisoning from failures
-    result = agent.invoke(
-        {"messages": [("human", text)]},
-        config={"configurable": {"thread_id": "session"}},
-    )
-    response = result["messages"][-1].content
-    print(f">>> Response ({len(response)} chars)")
-
-    # Replace progress bubble with actual response
-    chat_history[-1] = {"role": "assistant", "content": response}
-    yield chat_history, "", _latest_output()
-
-
-# ═══════════════════════════════════════════════
-# UI LAYOUT — Everything fits ONE screen (~500px)
-#
-# ┌────────────────────────────────────┐
-# │ 🔬 BERTopic Agent (compact)       │  ~55px
-# ├────────────────────────────────────┤
-# │ [📂 Upload CSV]                   │  ~35px
-# ├────────────────────────────────────┤
-# │ 💬 Chat bubbles (cloud style)     │
-# │    User: "Analyze my data"        │  ~320px
-# │    Agent: "I found 1390 papers.   │
-# │     Which config? All 3?"         │
-# ├────────────────────────────────────┤
-# │ [Type message...          ] [⏎]   │  ~35px
-# ├────────────────────────────────────┤
-# │ 📥 Download                       │  ~35px
-# └────────────────────────────────────┘
-#   TOTAL: ~480px — fits any screen
-# ═══════════════════════════════════════════════
-print(">>> Building UI...")
-with gr.Blocks(title="Topic Modelling — Agentic AI") as demo:
-    gr.HTML(HEADER_HTML)
-
-    # ══════════════════════════════════════════
-    # SECTION 1: DATA INPUT
-    # ══════════════════════════════════════════
-    gr.HTML('<div class="section-label">① DATA INPUT</div>')
-    with gr.Group():
-        with gr.Row():
-            upload = gr.File(label="���� Upload Scopus CSV", file_types=[".csv"])
-            gr.Markdown("**Upload your CSV** then type `run abstract only` in the chat below")
-
-    # ══════════════════════════════════════════
-    # PHASE PROGRESS PIPELINE
-    # ══════════════════════════════════════════
-    phase_progress = gr.HTML(value=_build_progress())
-
-    # ══════════════════════════════════════════
-    # SECTION 2: AGENT CONVERSATION
-    # ══════════════════════════════════════════
-    gr.HTML('<div class="section-label">② AGENT CONVERSATION — follow the prompts below</div>')
-    with gr.Group():
-        chatbot = gr.Chatbot(height=250, show_label=False,
-                             placeholder="Upload your Scopus CSV above, then type: run abstract only")
-        with gr.Row():
-            msg = gr.Textbox(
-                placeholder="run · approve · show topic 4 papers · group 0 1 5 · done",
-                show_label=False, scale=9, lines=1, max_lines=1, container=False)
-            send = gr.Button("Send", variant="primary", scale=1, min_width=70)
-
-    # ══════════════════════════════════════════
-    # SECTION 3: RESULTS
-    # ══════════════════════════════════════════
-    gr.HTML('<div class="section-label">③ RESULTS — review table, charts, downloads</div>')
-    with gr.Group():
-        with gr.Tabs():
-            with gr.Tab("📋 Review Table"):
-                gr.Markdown("*Edit Approve / Rename To / Your Reasoning → click Submit. "
-                            "Type `show topic 4 papers` in chat to see paper list.*")
-                review_table = gr.Dataframe(
-                    headers=["#", "Topic Label", "Top Evidence Sentence",
-                             "Sentences", "Papers", "Approve", "Rename To", "Your Reasoning"],
-                    datatype=["number", "str", "str", "number", "number", "str", "str", "str"],
-                    column_count=(8, "fixed"),
-                    interactive=True,
-                    row_count=(1, "dynamic"),
-                )
-                submit_review = gr.Button("✅ Submit Review to Agent", variant="primary")
-
-                # Paper viewer — select topic to see papers
-                gr.Markdown("---")
-                gr.Markdown("**📄 View papers in a topic:**")
-                with gr.Row():
-                    topic_num = gr.Number(label="Topic #", value=0, precision=0, minimum=0, scale=1)
-                    view_papers_btn = gr.Button("Show Papers", scale=1)
-                paper_list = gr.Textbox(label="Papers in selected topic", lines=8,
-                                        interactive=False)
-
-            with gr.Tab("📊 Charts"):
-                chart_selector = gr.Dropdown(choices=[], label="Select Chart", interactive=True)
-                chart_display = gr.HTML(
-                    value="<div style='height:350px;display:flex;align-items:center;justify-content:center;"
-                          "color:#94a3b8;border:1px dashed #cbd5e1;border-radius:8px;'>"
-                          "Charts appear after BERTopic runs</div>")
-
-            with gr.Tab("📥 Download"):
-                gr.Markdown(
-                    "**Files by Phase (per run: abstract / title):**\n\n"
-                    "**Phase 2 — Discovery:** `summaries.json` (raw topics) · `emb.npy` (embeddings)\n\n"
-                    "**Phase 2 — Labeling:** `labels.json` (Mistral-labeled topics)\n\n"
-                    "**Phase 2 — Charts:** `intertopic.html` · `bars.html` · `hierarchy.html` · `heatmap.html`\n\n"
-                    "**Phase 3 — Themes:** `themes.json` (consolidated themes)\n\n"
-                    "**Phase 5.5 — Taxonomy:** `taxonomy_map.json` (PAJAIS mapped vs NOVEL)\n\n"
-                    "**Phase 6 — Report:** `comparison.csv` (abstract vs title) · `narrative.txt` (500-word draft)"
-                )
-                download = gr.File(label="All output files", file_count="multiple")
-
-    def _load_chart(chart_name):
-        """Load chart HTML into iframe — gr.HTML strips scripts, iframe executes them."""
-        import html as html_mod
-        path = f"/tmp/{chart_name}"
-        content = os.path.exists(path) and open(path).read() or ""
-        escaped = html_mod.escape(content) * bool(content)
-        return (f'<iframe srcdoc="{escaped}" width="100%" height="450" '
-                f'frameborder="0" sandbox="allow-scripts allow-same-origin"></iframe>'
-                ) * bool(escaped) or "<div style='padding:20px;color:#94a3b8;'>Select a chart above</div>"
-
-    def _get_chart_choices():
-        """Find all rq4_*.html chart files in /tmp."""
-        files = sorted(glob.glob("/tmp/rq4_*.html"))
-        return list(map(os.path.basename, files))
-
-    def _load_review_table():
-        """Load review table from latest phase JSON.
-        Priority: taxonomy_map > themes > labels > summaries.
-        Returns 8-column rows: #, Label, Evidence/Mapping, Sent, Papers, Approve, Rename, Reasoning"""
-        import json
-        taxonomy_files = sorted(glob.glob("/tmp/checkpoints/rq4_*_taxonomy_map.json"))
-        theme_files = sorted(glob.glob("/tmp/checkpoints/rq4_*_themes.json"))
-        label_files = sorted(glob.glob("/tmp/checkpoints/rq4_*_labels.json"))
-        summary_files = sorted(glob.glob("/tmp/checkpoints/rq4_*_summaries.json"))
-
-        # Determine which file to load
-        path = ((taxonomy_files and taxonomy_files[-1])
-                or (theme_files and theme_files[-1])
-                or (label_files and label_files[-1])
-                or (summary_files and summary_files[-1]) or "")
-        is_taxonomy = bool(taxonomy_files and taxonomy_files[-1] == path)
-        data = (os.path.exists(path) and json.load(open(path))) or []
-
-        # For taxonomy: merge with themes to get sentence/paper counts
-        theme_lookup = {}
-        (is_taxonomy and theme_files) and theme_lookup.update({
-            t.get("label", ""): t for t in json.load(open(theme_files[-1]))})
-
-        rows = list(map(
-            lambda pair: [
-                pair[0],
-                pair[1].get("label", pair[1].get("top_words", "")[:60]),
-                # Evidence column: show PAJAIS mapping for taxonomy, evidence sentence otherwise
-                (is_taxonomy and f"→ {pair[1].get('pajais_match', '?')} | {pair[1].get('reasoning', '')}"[:120])
-                or (pair[1].get("nearest", [{}])[0].get("sentence", "")[:120] + "...") * bool(pair[1].get("nearest")),
-                # Sentence/paper counts: from taxonomy lookup or direct
-                theme_lookup.get(pair[1].get("label", ""), pair[1]).get("sentence_count", pair[1].get("sentence_count", 0)),
-                theme_lookup.get(pair[1].get("label", ""), pair[1]).get("paper_count", pair[1].get("paper_count", 0)),
-                "yes",
-                "",
-                "",
-            ], enumerate(data)))
-        return rows or [[0, "No data yet", "", 0, 0, "", "", ""]]
-
-    def _show_papers(topic_id):
-        """Show 5 nearest centroid sentences (evidence) + all paper titles for selected topic."""
-        import json
-        topic_id = int(topic_id)
-        summary_files = sorted(glob.glob("/tmp/checkpoints/rq4_*_summaries.json"))
-        label_files = sorted(glob.glob("/tmp/checkpoints/rq4_*_labels.json"))
-        all_files = label_files or summary_files
-        source_labels = list(map(
-            lambda f: os.path.basename(f).split("_")[1], all_files))
-        all_data = list(map(lambda f: json.load(open(f)), all_files))
-
-        lines = []
-        list(map(lambda pair: list(map(
-            lambda t: (t.get("topic_id") == topic_id) and lines.append(
-                f"═══ {pair[0].upper()} — Topic {topic_id}: "
-                f"{t.get('label', t.get('top_words','')[:50])} ═══\n"
-                f"{t.get('sentence_count', 0)} sentences from {t.get('paper_count', 0)} papers\n"
-                f"AI Reasoning: {t.get('reasoning', 'not yet labeled')}\n\n"
-                f"── 5 NEAREST CENTROID SENTENCES (evidence) ──\n"
-                + "\n".join(list(map(
-                    lambda i: f"  {i+1}. \"{t['nearest'][i]['sentence'][:200]}\"\n"
-                              f"     Paper: {t['nearest'][i].get('title', '')[:100]}",
-                    range(min(5, len(t.get('nearest', [])))))))
-                + "\n\n── ALL PAPER TITLES ──\n"
-                + "\n".join(list(map(
-                    lambda i: f"  {i+1}. {t['paper_titles'][i]}",
-                    range(len(t.get('paper_titles', []))))))
-            ),
-            pair[1])),
-            zip(source_labels, all_data)))
-
-        return "\n\n".join(lines) or f"Topic {topic_id} not found."
-
-    view_papers_btn.click(_show_papers, [topic_num], [paper_list])
-
-    def _submit_review(table_data, chat_history):
-        """Convert edited review table into a message for the agent.
-        Researcher's edits (approve/rename/reasoning) become natural language."""
-        rows = table_data.values.tolist()
-        lines = list(map(
-            lambda r: (
-                f"Topic {int(r[0])}: "
-                + (f"RENAME to '{r[6]}'" * bool(str(r[6]).strip()))
-                + (f"APPROVE '{r[1]}'" * (not bool(str(r[6]).strip())) * (str(r[5]).lower().startswith("y")))
-                + (f"REJECT" * (str(r[5]).lower().startswith("n")))
-                + (f" — reason: {r[7]}" * bool(str(r[7]).strip()))
-            ), rows))
-        review_msg = "Review decisions:\n" + "\n".join(lines)
-        print(f">>> Review submitted: {review_msg[:200]}")
-
-        chat_history = chat_history + [
-            {"role": "user", "content": review_msg},
-            {"role": "assistant", "content": "🔬 **Processing review decisions...**"},
-        ]
-        yield chat_history, _latest_output(), gr.update(), gr.update(), gr.update(), _build_progress()
-
-        result = agent.invoke(
-            {"messages": [("human", review_msg)]},
-            config={"configurable": {"thread_id": "session"}},
-        )
-        response = result["messages"][-1].content
-        chat_history[-1] = {"role": "assistant", "content": response}
-        # Reload table with updated themes/labels
-        table_data = _load_review_table()
-        yield (chat_history, _latest_output(),
-               gr.update(choices=_get_chart_choices()), gr.update(),
-               gr.update(value=table_data), _build_progress())
-
-    chart_selector.change(_load_chart, [chart_selector], [chart_display])
-    submit_review.click(_submit_review, [review_table, chatbot],
-                        [chatbot, download, chart_selector, chart_display, review_table, phase_progress])
-
-    def respond_with_viz(message, chat_history, uploaded_file):
-        """Wrap respond() and update chart dropdown + review table after each message."""
-        gen = respond(message, chat_history, uploaded_file)
-        # First yield (progress)
-        hist, txt, dl = next(gen)
-        yield hist, txt, dl, gr.update(choices=_get_chart_choices()), gr.update(), gr.update(), _build_progress()
-        # Second yield (final response + populate table + charts)
-        hist, txt, dl = next(gen)
-        choices = _get_chart_choices()
-        first_chart = (choices and _load_chart(choices[-1])) or gr.update()
-        table_data = _load_review_table()
-        yield (hist, txt, dl,
-               gr.update(choices=choices, value=(choices and choices[-1]) or None),
-               first_chart,
-               gr.update(value=table_data), _build_progress())
-
-    msg.submit(respond_with_viz, [msg, chatbot, upload],
-               [chatbot, msg, download, chart_selector, chart_display, review_table, phase_progress])
-    send.click(respond_with_viz, [msg, chatbot, upload],
-               [chatbot, msg, download, chart_selector, chart_display, review_table, phase_progress])
-
-    def _auto_load_csv(uploaded_file, chat_history):
-        """Auto-trigger analysis when CSV is uploaded — stats appear without typing."""
-        gen = respond("Analyze my Scopus CSV", chat_history, uploaded_file)
-        hist, txt, dl = next(gen)
-        yield hist, dl, gr.update(), gr.update(), gr.update(), _build_progress()
-        hist, txt, dl = next(gen)
-        choices = _get_chart_choices()
-        first_chart = (choices and _load_chart(choices[-1])) or gr.update()
-        table_data = _load_review_table()
-        yield (hist, dl,
-               gr.update(choices=choices, value=(choices and choices[-1]) or None),
-               first_chart,
-               gr.update(value=table_data), _build_progress())
-
-    upload.change(_auto_load_csv, [upload, chatbot],
-                  [chatbot, download, chart_selector, chart_display, review_table, phase_progress])
-
-print(">>> Launching...")
-demo.launch(server_name="0.0.0.0", server_port=7860, ssr_mode=False)