Upload 2 files

agent.py

@@ -0,0 +1,159 @@
+"""
+agent.py — Braun & Clarke (2006) Thematic Analysis Agent.
+
+10 tools. 6 STOP gates. Reviewer approval after every interpretive output.
+Every number comes from a tool — the LLM never computes values.
+"""
+
+from langchain_mistralai import ChatMistralAI
+from langchain.agents import create_agent
+from langgraph.checkpoint.memory import InMemorySaver
+from tools import ALL_TOOLS
+
+SYSTEM_PROMPT = """
+You are a Braun & Clarke (2006) Computational Thematic Analysis Agent.
+
+RULES:
+  1. ONE PHASE PER MESSAGE — STRICTLY ENFORCED.
+     After calling a tool, IMMEDIATELY present results and STOP.
+     Do NOT call a second tool in the same message.
+     Do NOT skip ahead to the next phase.
+     Do NOT combine phases.
+     The sequence MUST be: call tool → summarise result → STOP → wait.
+     Example CORRECT flow:
+       Message 1: Call load_scopus_csv → "Loaded 1,390 papers" → STOP
+       Message 2: Call run_bertopic_discovery → "Found 98 clusters" → STOP
+       Message 3: Call label_topics_with_llm → "Labelled 98 clusters" → STOP
+     Example WRONG flow:
+       Message 1: Call load_scopus_csv → call run_bertopic_discovery →
+                  call label_topics_with_llm → "All done!" ← NEVER DO THIS
+  2. ALL APPROVALS VIA REVIEW TABLE — never via chat. When review needed:
+       [WAITING FOR REVIEW TABLE]
+       Edit Approve / Rename To / Move To / Reasoning, then Submit Review.
+  3. NEVER FABRICATE DATA — every number, percentage, score, sentence list
+     MUST come from a tool. You CANNOT do arithmetic. If you need a number,
+     call a tool. If no tool exists for what you need, say so.
+  4. STOP GATES ARE ABSOLUTE — [FAILED] halts unconditionally.
+  5. EMIT PHASE STATUS at top of every response:
+       "[Phase X/6 | STOP Gates Passed: N/6 | Pending Review: Yes/No]"
+  6. TOOL ERRORS: log verbatim, identify cause, propose fix, wait.
+  7. AUTHOR KEYWORDS EXCLUDED from all embedding and clustering.
+  8. CHAT IS CONVERSATION, NOT DATA DUMP.
+     Your response in the chat window must be SHORT and CONVERSATIONAL:
+       - 3-5 sentences maximum summarising what you did
+       - State key numbers: "Found 45 clusters, 12 orphans"
+       - NEVER put markdown tables, JSON, raw data, or long lists in chat
+       - NEVER repeat the full tool output in chat
+     The Review Table (Section 3) auto-populates from your tool's
+     checkpoint files. The user sees the data THERE, not in chat.
+
+     REVIEW TABLE STATUS — say the right thing for the right phase:
+       - PHASE 1 (load_scopus_csv): NO review table data exists yet.
+         End with: "Type 'run abstract' or 'run title' to proceed to
+         BERTopic discovery." Do NOT say "Results in Review Table."
+       - PHASE 2+ (after run_bertopic_discovery, label_topics_with_llm,
+         consolidate_into_themes, etc.): Review table IS populated.
+         End with: "Results are loaded in the Review Table below.
+         Please review and click Submit Review when ready."
+     The rule: only mention the Review Table if your tool actually
+     wrote a JSON checkpoint file (topic_labels.json, themes.json,
+     summaries.json, taxonomy_alignment.json) that the table can load.
+
+10 TOOLS:
+  DETERMINISTIC (same input → same output):
+    1.  load_scopus_csv          — Phase 1: clean CSV, count, save .parquet
+    2.  run_bertopic_discovery   — Phase 2: embed + cluster (min 3 members)
+                                    + orphan report + 4 charts
+    4.  reassign_sentences       — Phase 2: move orphans/sentences between clusters
+    5.  consolidate_into_themes  — Phase 3: merge groups, recompute centroids
+    6.  compute_saturation       — Phase 4: coverage %, coherence, balance
+    7.  generate_theme_profiles  — Phase 5: top 5 nearest sentences per theme
+    9.  generate_comparison_csv  — Phase 6: abstract vs title joined on PAJAIS
+
+  LLM-DEPENDENT (grounded in real data, reviewer must approve):
+    3.  label_topics_with_llm    — Phase 2: Mistral names clusters
+    8.  compare_with_taxonomy    — Phase 5.5: map themes to PAJAIS 25
+    10. export_narrative         — Phase 6: 500-word Section 7
+
+B&C 6-PHASE METHODOLOGY:
+
+  PHASE 1 — FAMILIARISATION
+    The user message may contain a [CSV: /path/to/file.csv] prefix.
+    Extract the FULL path (everything between "CSV: " and "]") and pass
+    it as csv_path to load_scopus_csv. Do NOT modify or shorten the path.
+    Call load_scopus_csv. Show stats. STOP. Wait for "run abstract"/"run title".
+
+  PHASE 2 — INITIAL CODES (3 separate messages, one tool each)
+    MESSAGE 1: Call run_bertopic_discovery. Report: total clusters, orphan count.
+    Say "Results loaded in the Review Table below." STOP. Wait.
+    MESSAGE 2 (after user says proceed): Call label_topics_with_llm.
+    Report: how many labelled. Say "Labels loaded in Review Table." STOP.
+    If orphans > 0, tell reviewer: "N sentences did not fit any cluster
+    (minimum 3 members required). Use Move To column to reassign."
+    STOP GATE 1: SG1-A (<5 topics), SG1-B (confidence <0.40),
+                 SG1-C (>40% generic), SG1-D (duplicates).
+    [WAITING FOR REVIEW TABLE]. STOP.
+    MESSAGE 3 (after Submit Review): if moves exist, call reassign_sentences.
+
+  PHASE 3 — THEMES
+    Parse review. Call consolidate_into_themes.
+    STOP GATE 2: SG2-A (<3 themes), SG2-B (singleton),
+                 SG2-C (duplicates), SG2-D (coverage <50%).
+    [WAITING FOR REVIEW TABLE]. STOP.
+
+  PHASE 4 — SATURATION
+    Call compute_saturation (NEVER compute these numbers yourself).
+    Present the EXACT numbers returned by the tool.
+    STOP GATE 3: SG3-A (coverage <60%), SG3-B (single theme >60%),
+                 SG3-C (coherence <0.30), SG3-D (<3 themes).
+    [WAITING FOR REVIEW TABLE]. STOP.
+
+  PHASE 5 — NAMING
+    Call generate_theme_profiles (NEVER recall sentences from memory).
+    Present the EXACT top-5 sentences returned by the tool per theme.
+    Propose names based on these real sentences.
+    [WAITING FOR REVIEW TABLE]. STOP.
+
+  PHASE 5.5 — PAJAIS MAPPING
+    Call compare_with_taxonomy.
+    STOP GATE 4: SG4-A (zero categories), SG4-B (>30% score <0.40),
+                 SG4-C (single category >50%), SG4-D (incomplete).
+    [WAITING FOR REVIEW TABLE]. STOP.
+
+  PHASE 6 — REPORT
+    Call generate_comparison_csv. Present convergence/divergence summary.
+    STOP GATE 5: Reviewer confirms comparison makes sense.
+    [WAITING FOR REVIEW TABLE]. STOP.
+    Call export_narrative. Present full 500-word draft.
+    STOP GATE 6: Reviewer approves final narrative.
+    [WAITING FOR REVIEW TABLE]. STOP.
+    DONE — all 6 gates passed.
+
+6 STOP GATES:
+  STOP-1 (Phase 2)   : Initial Code Quality
+  STOP-2 (Phase 3)   : Theme Coherence
+  STOP-3 (Phase 4)   : Saturation Adequacy
+  STOP-4 (Phase 5.5) : Taxonomy Alignment Quality
+  STOP-5 (Phase 6)   : Comparison Review         [NEW]
+  STOP-6 (Phase 6)   : Narrative Approval         [NEW]
+"""
+
+llm = ChatMistralAI(model="mistral-large-latest", temperature=0, max_tokens=8192)
+
+memory = InMemorySaver()
+
+agent = create_agent(
+    model=llm,
+    tools=ALL_TOOLS,
+    system_prompt=SYSTEM_PROMPT,
+    checkpointer=memory,
+)
+
+
+def run(user_message: str, thread_id: str = "default") -> str:
+    """Invoke the agent for one conversation turn."""
+    config  = {"configurable": {"thread_id": thread_id}}
+    payload = {"messages": [{"role": "user", "content": user_message}]}
+    result  = agent.invoke(payload, config=config)
+    msgs    = result.get("messages", [])
+    return (msgs and msgs[-1].content) or ""

app.py

@@ -0,0 +1,326 @@
+"""
+app.py — BERTopic Topic Modelling Agent UI.
+
+Three UX features:
+    1. Phase banner — large prominent display of current B&C phase
+    2. Dynamic prompts — phase-appropriate suggested next actions
+    3. Auto-populated review table — loads from tool checkpoint files
+
+9-column review table: #, Topic Label, Top Evidence, Sentences, Papers,
+Approve, Rename To, Move To, Reasoning.
+"""
+
+import gradio as gr
+import pandas as pd
+import json
+import os
+import re
+import tempfile
+from datetime import datetime
+from pathlib import Path
+from agent import run as agent_run
+
+THREAD_ID = f"bertopic-{datetime.now().strftime('%Y%m%d%H%M%S')}"
+
+REVIEW_COLS = [
+    "#", "Topic Label", "Top Evidence", "Sentences", "Papers",
+    "Approve", "Rename To", "Move To", "Reasoning",
+]
+
+EMPTY_TABLE = pd.DataFrame(
+    {"#": ["-"], "Topic Label": ["No results yet — run analysis first"],
+     "Top Evidence": [""], "Sentences": [""], "Papers": [""],
+     "Approve": [""], "Rename To": [""], "Move To": [""], "Reasoning": [""]},
+)
+
+PHASE_INFO = {
+    0: ("Getting started", "⬜⬜⬜⬜⬜⬜",
+        "Upload a CSV file, then click **Analyze my Scopus CSV** and press Send"),
+    1: ("Phase 1 — Familiarisation", "🟦⬜⬜⬜⬜⬜",
+        "Click **Run abstract analysis** or **Run title analysis** and press Send"),
+    2: ("Phase 2 — Initial Codes", "🟦🟦⬜⬜⬜⬜",
+        "Review clusters in the **Results table** below. Edit Approve / Rename / Move, "
+        "then click **Submit Review**"),
+    3: ("Phase 3 — Themes", "🟦🟦🟦⬜⬜⬜",
+        "Review merged themes. Edit the table, then click **Submit Review**"),
+    4: ("Phase 4 — Saturation", "🟦🟦🟦🟦⬜⬜",
+        "Review saturation metrics. Click **Submit Review** to confirm"),
+    5: ("Phase 5 — Naming", "🟦🟦🟦🟦🟦⬜",
+        "Review theme profiles. Edit names, then **Submit Review**"),
+    6: ("Phase 6 — Report", "🟦🟦🟦🟦🟦🟦",
+        "Review comparison and narrative. **Submit Review** to finalise"),
+}
+
+_path = lambda file: str(
+    (hasattr(file, "name") and file.name)
+    or (isinstance(file, str) and file)
+    or ""
+)
+_name = lambda file: os.path.basename(_path(file))
+
+
+def _extract_phase(text: str) -> int:
+    """Extract phase number from agent response. Returns 0 if not found."""
+    found = re.findall(r"Phase (\d)", str(text))
+    return int((found or ["0"])[0])
+
+
+def _phase_banner(num: int) -> str:
+    """Generate prominent phase banner with progress bar and next step."""
+    name, progress, instruction = PHASE_INFO.get(num, PHASE_INFO[0])
+    return (
+        f"## {progress}  {name}\n\n"
+        f"**NEXT STEP →** {instruction}"
+    )
+
+
+def _load_review_table(base_dir: str) -> pd.DataFrame:
+    """Load latest checkpoint file into the 9-column review table.
+
+    Scans base_dir for topic_labels.json, themes.json, taxonomy_alignment.json,
+    summaries.json. Loads the most recently modified one and formats it.
+    Returns EMPTY_TABLE if nothing found.
+    """
+    base = Path(str(base_dir or "/tmp/nonexistent_dir_placeholder"))
+    candidates = (
+        base_dir and base.exists() and sorted(
+            (
+                list(base.glob("topic_labels.json"))
+                + list(base.glob("themes.json"))
+                + list(base.glob("taxonomy_alignment.json"))
+                + list(base.glob("summaries.json"))
+            ),
+            key=lambda p: p.stat().st_mtime,
+            reverse=True,
+        )
+    ) or []
+
+    latest = (candidates[:1] or [None])[0]
+    return (latest and [_format_checkpoint(latest)] or [EMPTY_TABLE.copy()])[0]
+
+
+def _format_checkpoint(path) -> pd.DataFrame:
+    """Format a checkpoint JSON file into review table rows.
+
+    Merges data from multiple checkpoint files when available:
+    topic_labels.json has labels but no sizes — summaries.json has sizes.
+    """
+    raw = json.loads(Path(path).read_text())
+    base = Path(path).parent
+
+    data = (isinstance(raw, dict) and raw.get("clusters", raw.get("per_theme", []))) or \
+           (isinstance(raw, list) and raw) or []
+
+    summaries_data = {}
+    summaries_path = base / "summaries.json"
+    summaries_raw = (
+        summaries_path.exists() and json.loads(summaries_path.read_text()) or {}
+    )
+    summaries_list = (
+        isinstance(summaries_raw, dict) and summaries_raw.get("clusters", [])
+    ) or (isinstance(summaries_raw, list) and summaries_raw) or []
+    list(map(
+        lambda s: summaries_data.update({s.get("topic_id", -999): s}),
+        summaries_list,
+    ))
+
+    def _row(item: dict) -> dict:
+        """Map one JSON item to review table columns, merging summaries data."""
+        tid = item.get("topic_id", item.get("theme_id", 0))
+        summary = summaries_data.get(tid, {})
+        return {
+            "#":            tid,
+            "Topic Label":  item.get("label", item.get("theme_label", "")),
+            "Top Evidence":  str(
+                item.get("representative", "")
+                or summary.get("representative", "")
+                or item.get("notes", "")
+            )[:150],
+            "Sentences":    item.get("size", 0) or summary.get("size", 0)
+                            or item.get("total_papers", 0),
+            "Papers":       item.get("size", 0) or summary.get("size", 0)
+                            or item.get("total_papers", 0),
+            "Approve":      "Yes",
+            "Rename To":    "",
+            "Move To":      "",
+            "Reasoning":    str(item.get("rationale", item.get("notes", ""))),
+        }
+
+    rows = list(map(_row, data[:200]))
+    return (rows and [pd.DataFrame(rows, columns=REVIEW_COLS)] or [EMPTY_TABLE.copy()])[0]
+
+
+def on_file_upload(file):
+    """Extract CSV stats and store base directory."""
+    path = _path(file)
+    result = (not path) and ("Upload a CSV to begin.", "", _phase_banner(0))
+    return result or _do_file_upload(path, file)
+
+
+def _do_file_upload(path: str, file) -> tuple:
+    """Actual file processing after path validation."""
+    df = pd.read_csv(path)
+    rows, cols = df.shape
+    base = str(Path(path).parent)
+    info = (
+        f"**Loaded:** `{_name(file)}`\n\n"
+        f"**Shape:** {rows:,} rows x {cols} columns\n\n"
+        f"**Columns:** {', '.join(df.columns[:6].tolist())}\n\n"
+        f"*Click a prompt below and press Send to begin.*"
+    )
+    return info, base, _phase_banner(1)
+
+
+def on_send(user_msg, history, file, base_dir):
+    """Pass user message to agent. Update phase banner and review table."""
+    msg = (user_msg or "").strip() or "help"
+    csv_tag = f"[CSV: {_path(file)}]\n" * bool(file)
+
+    history = list(history or [])
+    history.append({"role": "user", "content": msg})
+    history.append({"role": "assistant", "content": "Thinking..."})
+    yield history, "", gr.skip(), gr.skip(), gr.skip()
+
+    reply = agent_run(csv_tag + msg, thread_id=THREAD_ID)
+    history[-1] = {"role": "assistant", "content": reply}
+
+    phase = _extract_phase(reply)
+    banner = _phase_banner(phase)
+    table = _load_review_table(base_dir)
+
+    yield history, "", banner, table, base_dir
+
+
+def on_submit_review(table_df, history, base_dir):
+    """Serialise review table edits to agent."""
+    history = list(history or [])
+    edits = table_df.to_json(orient="records", indent=2)
+
+    history.append({"role": "user", "content": "[REVIEW SUBMITTED]"})
+    history.append({"role": "assistant", "content": "Processing review..."})
+
+    reply = agent_run(
+        "Reviewer submitted table edits.\n\n"
+        f"```json\n{edits}\n```\n\n"
+        "Process: Approve/Reject decisions, Rename To values, "
+        "Move To reassignments (call reassign_sentences if moves exist), "
+        "Reasoning notes. Then check STOP gates and proceed.",
+        thread_id=THREAD_ID,
+    )
+    history[-1] = {"role": "assistant", "content": reply}
+
+    phase = _extract_phase(reply)
+    return history, _phase_banner(phase), _load_review_table(base_dir)
+
+
+def on_download(table_df, history):
+    """Export review CSV and chat TXT."""
+    csv_tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".csv", prefix="review_")
+    table_df.to_csv(csv_tmp.name, index=False)
+
+    txt_tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".txt", prefix="chat_")
+    txt_tmp.write(
+        "\n\n".join(
+            list(map(
+                lambda m: f"{m.get('role', '').upper()}: {m.get('content', '')}",
+                history or [],
+            ))
+        ).encode("utf-8")
+    )
+    txt_tmp.close()
+    return [csv_tmp.name, txt_tmp.name]
+
+
+with gr.Blocks(title="BERTopic Agent") as demo:
+
+    base_dir_state = gr.State(value="")
+
+    gr.Markdown("# BERTopic Modelling Agent")
+    gr.Markdown(
+        "**Braun & Clarke 6-Phase Thematic Analysis** "
+        "| 10 Tools | 6 STOP Gates | Cosine Agglomerative Clustering"
+    )
+
+    phase_banner = gr.Markdown(value=_phase_banner(0))
+
+    gr.Markdown("---\n### Section 1 — Data input")
+    with gr.Row():
+        with gr.Column(scale=3):
+            file_input = gr.File(
+                label="Upload Scopus CSV",
+                file_types=[".csv"],
+                file_count="single",
+            )
+        with gr.Column(scale=5):
+            file_info = gr.Markdown("Upload a CSV to begin.")
+
+    gr.Markdown("---\n### Section 2 — Agent conversation")
+    chatbot = gr.Chatbot(label="BERTopic Agent", height=200)
+    with gr.Row():
+        msg_box = gr.Textbox(
+            placeholder="Type a message or click a prompt below, then press Send",
+            show_label=False, scale=7, lines=1,
+        )
+        send_btn = gr.Button("Send", variant="primary", scale=1)
+
+    gr.Examples(
+        examples=[
+            "Analyze my Scopus CSV",
+            "Run abstract analysis",
+            "Run title analysis",
+            "Proceed to next phase",
+            "Show corpus statistics",
+        ],
+        inputs=msg_box,
+        label="Quick prompts (click to fill, then press Send)",
+    )
+
+    gr.Markdown("---\n### Section 3 — Results (auto-populated from tool outputs)")
+    gr.Markdown(
+        "This table fills automatically when the agent runs tools. "
+        "Edit **Approve**, **Rename To**, **Move To**, **Reasoning** columns, "
+        "then click **Submit Review**."
+    )
+    review_table = gr.Dataframe(
+        value=EMPTY_TABLE,
+        headers=REVIEW_COLS,
+        datatype=["number", "str", "str", "number", "number",
+                  "str", "str", "str", "str"],
+        column_count=(9, "fixed"),
+        interactive=True,
+        wrap=True,
+        max_height=400,
+    )
+    with gr.Row():
+        clear_btn = gr.Button("Clear table", variant="secondary", scale=2)
+        sub_btn = gr.Button("Submit Review", variant="primary", scale=4)
+
+    with gr.Accordion("Download", open=False):
+        dl_btn = gr.Button("Generate downloads", variant="primary")
+        dl_files = gr.File(label="Downloads", file_count="multiple",
+                          interactive=False)
+
+    file_input.change(
+        on_file_upload,
+        inputs=[file_input],
+        outputs=[file_info, base_dir_state, phase_banner],
+    )
+    send_btn.click(
+        on_send,
+        inputs=[msg_box, chatbot, file_input, base_dir_state],
+        outputs=[chatbot, msg_box, phase_banner, review_table, base_dir_state],
+    )
+    msg_box.submit(
+        on_send,
+        inputs=[msg_box, chatbot, file_input, base_dir_state],
+        outputs=[chatbot, msg_box, phase_banner, review_table, base_dir_state],
+    )
+    clear_btn.click(lambda: EMPTY_TABLE.copy(), outputs=[review_table])
+    sub_btn.click(
+        on_submit_review,
+        inputs=[review_table, chatbot, base_dir_state],
+        outputs=[chatbot, phase_banner, review_table],
+    )
+    dl_btn.click(on_download, inputs=[review_table, chatbot], outputs=[dl_files])
+
+demo.launch(ssr_mode=False, theme=gr.themes.Soft())