Upload app.py

app.py

@@ -0,0 +1,326 @@
+"""
+app.py — BERTopic Topic Modelling Agent UI.
+
+Three UX features:
+    1. Phase banner — large prominent display of current B&C phase
+    2. Dynamic prompts — phase-appropriate suggested next actions
+    3. Auto-populated review table — loads from tool checkpoint files
+
+9-column review table: #, Topic Label, Top Evidence, Sentences, Papers,
+Approve, Rename To, Move To, Reasoning.
+"""
+
+import gradio as gr
+import pandas as pd
+import json
+import os
+import re
+import tempfile
+from datetime import datetime
+from pathlib import Path
+from agent import run as agent_run
+
+THREAD_ID = f"bertopic-{datetime.now().strftime('%Y%m%d%H%M%S')}"
+
+REVIEW_COLS = [
+    "#", "Topic Label", "Top Evidence", "Sentences", "Papers",
+    "Approve", "Rename To", "Move To", "Reasoning",
+]
+
+EMPTY_TABLE = pd.DataFrame(
+    {"#": ["-"], "Topic Label": ["No results yet — run analysis first"],
+     "Top Evidence": [""], "Sentences": [""], "Papers": [""],
+     "Approve": [""], "Rename To": [""], "Move To": [""], "Reasoning": [""]},
+)
+
+PHASE_INFO = {
+    0: ("Getting started", "⬜⬜⬜⬜⬜⬜",
+        "Upload a CSV file, then click **Analyze my Scopus CSV** and press Send"),
+    1: ("Phase 1 — Familiarisation", "🟦⬜⬜⬜⬜⬜",
+        "Click **Run abstract analysis** or **Run title analysis** and press Send"),
+    2: ("Phase 2 — Initial Codes", "🟦🟦⬜⬜⬜⬜",
+        "Review clusters in the **Results table** below. Edit Approve / Rename / Move, "
+        "then click **Submit Review**"),
+    3: ("Phase 3 — Themes", "🟦🟦🟦⬜⬜⬜",
+        "Review merged themes. Edit the table, then click **Submit Review**"),
+    4: ("Phase 4 — Saturation", "🟦🟦🟦🟦⬜⬜",
+        "Review saturation metrics. Click **Submit Review** to confirm"),
+    5: ("Phase 5 — Naming", "🟦🟦🟦🟦🟦⬜",
+        "Review theme profiles. Edit names, then **Submit Review**"),
+    6: ("Phase 6 — Report", "🟦🟦🟦🟦🟦🟦",
+        "Review comparison and narrative. **Submit Review** to finalise"),
+}
+
+_path = lambda file: str(
+    (hasattr(file, "name") and file.name)
+    or (isinstance(file, str) and file)
+    or ""
+)
+_name = lambda file: os.path.basename(_path(file))
+
+
+def _extract_phase(text: str) -> int:
+    """Extract phase number from agent response. Returns 0 if not found."""
+    found = re.findall(r"Phase (\d)", str(text))
+    return int((found or ["0"])[0])
+
+
+def _phase_banner(num: int) -> str:
+    """Generate prominent phase banner with progress bar and next step."""
+    name, progress, instruction = PHASE_INFO.get(num, PHASE_INFO[0])
+    return (
+        f"## {progress}  {name}\n\n"
+        f"**NEXT STEP →** {instruction}"
+    )
+
+
+def _load_review_table(base_dir: str) -> pd.DataFrame:
+    """Load latest checkpoint file into the 9-column review table.
+
+    Scans base_dir for topic_labels.json, themes.json, taxonomy_alignment.json,
+    summaries.json. Loads the most recently modified one and formats it.
+    Returns EMPTY_TABLE if nothing found.
+    """
+    base = Path(str(base_dir or "/tmp/nonexistent_dir_placeholder"))
+    candidates = (
+        base_dir and base.exists() and sorted(
+            (
+                list(base.glob("topic_labels.json"))
+                + list(base.glob("themes.json"))
+                + list(base.glob("taxonomy_alignment.json"))
+                + list(base.glob("summaries.json"))
+            ),
+            key=lambda p: p.stat().st_mtime,
+            reverse=True,
+        )
+    ) or []
+
+    latest = (candidates[:1] or [None])[0]
+    return (latest and [_format_checkpoint(latest)] or [EMPTY_TABLE.copy()])[0]
+
+
+def _format_checkpoint(path) -> pd.DataFrame:
+    """Format a checkpoint JSON file into review table rows.
+
+    Merges data from multiple checkpoint files when available:
+    topic_labels.json has labels but no sizes — summaries.json has sizes.
+    """
+    raw = json.loads(Path(path).read_text())
+    base = Path(path).parent
+
+    data = (isinstance(raw, dict) and raw.get("clusters", raw.get("per_theme", []))) or \
+           (isinstance(raw, list) and raw) or []
+
+    summaries_data = {}
+    summaries_path = base / "summaries.json"
+    summaries_raw = (
+        summaries_path.exists() and json.loads(summaries_path.read_text()) or {}
+    )
+    summaries_list = (
+        isinstance(summaries_raw, dict) and summaries_raw.get("clusters", [])
+    ) or (isinstance(summaries_raw, list) and summaries_raw) or []
+    list(map(
+        lambda s: summaries_data.update({s.get("topic_id", -999): s}),
+        summaries_list,
+    ))
+
+    def _row(item: dict) -> dict:
+        """Map one JSON item to review table columns, merging summaries data."""
+        tid = item.get("topic_id", item.get("theme_id", 0))
+        summary = summaries_data.get(tid, {})
+        return {
+            "#":            tid,
+            "Topic Label":  item.get("label", item.get("theme_label", "")),
+            "Top Evidence":  str(
+                item.get("representative", "")
+                or summary.get("representative", "")
+                or item.get("notes", "")
+            )[:150],
+            "Sentences":    item.get("size", 0) or summary.get("size", 0)
+                            or item.get("total_papers", 0),
+            "Papers":       item.get("size", 0) or summary.get("size", 0)
+                            or item.get("total_papers", 0),
+            "Approve":      "Yes",
+            "Rename To":    "",
+            "Move To":      "",
+            "Reasoning":    str(item.get("rationale", item.get("notes", ""))),
+        }
+
+    rows = list(map(_row, data[:200]))
+    return (rows and [pd.DataFrame(rows, columns=REVIEW_COLS)] or [EMPTY_TABLE.copy()])[0]
+
+
+def on_file_upload(file):
+    """Extract CSV stats and store base directory."""
+    path = _path(file)
+    result = (not path) and ("Upload a CSV to begin.", "", _phase_banner(0))
+    return result or _do_file_upload(path, file)
+
+
+def _do_file_upload(path: str, file) -> tuple:
+    """Actual file processing after path validation."""
+    df = pd.read_csv(path)
+    rows, cols = df.shape
+    base = str(Path(path).parent)
+    info = (
+        f"**Loaded:** `{_name(file)}`\n\n"
+        f"**Shape:** {rows:,} rows x {cols} columns\n\n"
+        f"**Columns:** {', '.join(df.columns[:6].tolist())}\n\n"
+        f"*Click a prompt below and press Send to begin.*"
+    )
+    return info, base, _phase_banner(1)
+
+
+def on_send(user_msg, history, file, base_dir):
+    """Pass user message to agent. Update phase banner and review table."""
+    msg = (user_msg or "").strip() or "help"
+    csv_tag = f"[CSV: {_path(file)}]\n" * bool(file)
+
+    history = list(history or [])
+    history.append({"role": "user", "content": msg})
+    history.append({"role": "assistant", "content": "Thinking..."})
+    yield history, "", gr.skip(), gr.skip(), gr.skip()
+
+    reply = agent_run(csv_tag + msg, thread_id=THREAD_ID)
+    history[-1] = {"role": "assistant", "content": reply}
+
+    phase = _extract_phase(reply)
+    banner = _phase_banner(phase)
+    table = _load_review_table(base_dir)
+
+    yield history, "", banner, table, base_dir
+
+
+def on_submit_review(table_df, history, base_dir):
+    """Serialise review table edits to agent."""
+    history = list(history or [])
+    edits = table_df.to_json(orient="records", indent=2)
+
+    history.append({"role": "user", "content": "[REVIEW SUBMITTED]"})
+    history.append({"role": "assistant", "content": "Processing review..."})
+
+    reply = agent_run(
+        "Reviewer submitted table edits.\n\n"
+        f"```json\n{edits}\n```\n\n"
+        "Process: Approve/Reject decisions, Rename To values, "
+        "Move To reassignments (call reassign_sentences if moves exist), "
+        "Reasoning notes. Then check STOP gates and proceed.",
+        thread_id=THREAD_ID,
+    )
+    history[-1] = {"role": "assistant", "content": reply}
+
+    phase = _extract_phase(reply)
+    return history, _phase_banner(phase), _load_review_table(base_dir)
+
+
+def on_download(table_df, history):
+    """Export review CSV and chat TXT."""
+    csv_tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".csv", prefix="review_")
+    table_df.to_csv(csv_tmp.name, index=False)
+
+    txt_tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".txt", prefix="chat_")
+    txt_tmp.write(
+        "\n\n".join(
+            list(map(
+                lambda m: f"{m.get('role', '').upper()}: {m.get('content', '')}",
+                history or [],
+            ))
+        ).encode("utf-8")
+    )
+    txt_tmp.close()
+    return [csv_tmp.name, txt_tmp.name]
+
+
+with gr.Blocks(title="BERTopic Agent") as demo:
+
+    base_dir_state = gr.State(value="")
+
+    gr.Markdown("# BERTopic Modelling Agent")
+    gr.Markdown(
+        "**Braun & Clarke 6-Phase Thematic Analysis** "
+        "| 10 Tools | 6 STOP Gates | Cosine Agglomerative Clustering"
+    )
+
+    phase_banner = gr.Markdown(value=_phase_banner(0))
+
+    gr.Markdown("---\n### Section 1 — Data input")
+    with gr.Row():
+        with gr.Column(scale=3):
+            file_input = gr.File(
+                label="Upload Scopus CSV",
+                file_types=[".csv"],
+                file_count="single",
+            )
+        with gr.Column(scale=5):
+            file_info = gr.Markdown("Upload a CSV to begin.")
+
+    gr.Markdown("---\n### Section 2 — Agent conversation")
+    chatbot = gr.Chatbot(label="BERTopic Agent", height=200)
+    with gr.Row():
+        msg_box = gr.Textbox(
+            placeholder="Type a message or click a prompt below, then press Send",
+            show_label=False, scale=7, lines=1,
+        )
+        send_btn = gr.Button("Send", variant="primary", scale=1)
+
+    gr.Examples(
+        examples=[
+            "Analyze my Scopus CSV",
+            "Run abstract analysis",
+            "Run title analysis",
+            "Proceed to next phase",
+            "Show corpus statistics",
+        ],
+        inputs=msg_box,
+        label="Quick prompts (click to fill, then press Send)",
+    )
+
+    gr.Markdown("---\n### Section 3 — Results (auto-populated from tool outputs)")
+    gr.Markdown(
+        "This table fills automatically when the agent runs tools. "
+        "Edit **Approve**, **Rename To**, **Move To**, **Reasoning** columns, "
+        "then click **Submit Review**."
+    )
+    review_table = gr.Dataframe(
+        value=EMPTY_TABLE,
+        headers=REVIEW_COLS,
+        datatype=["number", "str", "str", "number", "number",
+                  "str", "str", "str", "str"],
+        column_count=(9, "fixed"),
+        interactive=True,
+        wrap=True,
+        max_height=400,
+    )
+    with gr.Row():
+        clear_btn = gr.Button("Clear table", variant="secondary", scale=2)
+        sub_btn = gr.Button("Submit Review", variant="primary", scale=4)
+
+    with gr.Accordion("Download", open=False):
+        dl_btn = gr.Button("Generate downloads", variant="primary")
+        dl_files = gr.File(label="Downloads", file_count="multiple",
+                          interactive=False)
+
+    file_input.change(
+        on_file_upload,
+        inputs=[file_input],
+        outputs=[file_info, base_dir_state, phase_banner],
+    )
+    send_btn.click(
+        on_send,
+        inputs=[msg_box, chatbot, file_input, base_dir_state],
+        outputs=[chatbot, msg_box, phase_banner, review_table, base_dir_state],
+    )
+    msg_box.submit(
+        on_send,
+        inputs=[msg_box, chatbot, file_input, base_dir_state],
+        outputs=[chatbot, msg_box, phase_banner, review_table, base_dir_state],
+    )
+    clear_btn.click(lambda: EMPTY_TABLE.copy(), outputs=[review_table])
+    sub_btn.click(
+        on_submit_review,
+        inputs=[review_table, chatbot, base_dir_state],
+        outputs=[chatbot, phase_banner, review_table],
+    )
+    dl_btn.click(on_download, inputs=[review_table, chatbot], outputs=[dl_files])
+
+demo.launch(ssr_mode=False, theme=gr.themes.Soft())