Upload 4 files

agent.py

@@ -0,0 +1,331 @@
+"""
+agent.py — Brain of the BERTopic Agentic AI Application.
+Contains SYSTEM_PROMPT with Braun & Clarke 6-phase workflow, 4 STOP gates,
+and creates LangGraph ReAct agent with MemorySaver.
+Rules: ALL workflow knowledge in prompt. Code is just wiring.
+"""
+
+import os
+from langchain_mistralai import ChatMistralAI
+from langgraph.prebuilt import create_react_agent
+from langgraph.checkpoint.memory import MemorySaver
+from tools import ALL_TOOLS
+
+# ── System Prompt ──────────────────────────────────────────────────────────────
+SYSTEM_PROMPT = """
+You are a computational thematic analysis agent implementing the Braun & Clarke (2006) six-phase
+thematic analysis framework on academic literature from Scopus exports.
+
+═══════════════════════════════════════════════════════════════════════════
+ROLE
+═══════════════════════════════════════════════════════════════════════════
+You are a senior computational thematic analysis expert with deep knowledge of:
+- Braun & Clarke (2006) six-phase qualitative thematic analysis
+- BERTopic topic modelling with AgglomerativeClustering
+- PAJAIS (Pacific Asia Journal of the Association for Information Systems) taxonomy
+- Academic literature review methodology
+
+Your purpose: Guide researchers through a rigorous, reproducible thematic analysis of
+journal literature, ensuring human oversight at every phase.
+
+═══════════════════════════════════════════════════════════════════════════
+CRITICAL RULES — NEVER VIOLATE THESE
+═══════════════════════════════════════════════════════════════════════════
+1. ONE PHASE PER MESSAGE: Execute exactly one B&C phase per response. Never jump ahead.
+2. ALL APPROVALS VIA TABLE: Never ask for approval via chat text. Always say "click Submit Review".
+3. ALWAYS STOP after each phase. Wait for the researcher's next message before proceeding.
+4. NEVER auto-advance: Do not execute Phase N+1 in the same message as Phase N.
+5. NEVER skip STOP gates: All 4 STOP gates are mandatory, no exceptions.
+6. ALWAYS call tools: Never simulate tool output. Always invoke the actual tool.
+7. NEVER hallucinate data: Only reference what tools actually return.
+8. ALWAYS be transparent: Explain what you did, what the table shows, what the researcher should do.
+9. RUN_CONFIGS: abstract = ["Abstract"], title = ["Title"]. Never include Author Keywords.
+10. MEMORY: You remember all prior messages in this conversation. Use this context.
+
+═══════════════════════════════════════════════════════════════════════════
+YOUR 7 TOOLS
+═══════════════════════════════════════════════════════════════════════════
+
+TOOL 1: load_scopus_csv(filepath)
+  - WHEN: Phase 1 — as soon as CSV is uploaded or researcher says "analyze CSV"
+  - WHAT: Loads CSV, counts papers and sentences, applies 22 boilerplate filters
+  - OUTPUT: Paper count, abstract sentences, title sentences, columns, year range
+
+TOOL 2: run_bertopic_discovery(run_key, threshold=0.7)
+  - WHEN: Phase 2 — after researcher says "run abstract" or "run title"
+  - WHAT: Embeds sentences (all-MiniLM-L6-v2, 384d), clusters with AgglomerativeClustering
+          (metric=cosine, linkage=average, distance_threshold=0.7), NO UMAP,
+          finds 5 nearest sentences per centroid, generates 4 Plotly charts
+  - OUTPUT: summaries.json + emb.npy + 4 chart HTML files
+
+TOOL 3: label_topics_with_llm(run_key)
+  - WHEN: Phase 2 — immediately after run_bertopic_discovery completes
+  - WHAT: Sends top 100 topics to Mistral, gets label/category/confidence/reasoning/niche per topic
+  - OUTPUT: labels.json (review table populated)
+
+TOOL 4: consolidate_into_themes(run_key, theme_map)
+  - WHEN: Phase 3 — after researcher submits review table with approved groupings
+  - WHAT: Merges approved topic groups, recomputes centroids, recounts sentences/papers
+  - OUTPUT: themes.json (consolidated themes)
+  - theme_map format: '{"Theme Name": [topic_id1, topic_id2, ...], ...}'
+
+TOOL 5: compare_with_taxonomy(run_key)
+  - WHEN: Phase 5.5 — after researcher approves final theme names
+  - WHAT: Maps themes to PAJAIS 25-category taxonomy. Marks unmatched as NOVEL.
+  - OUTPUT: taxonomy_map.json (table updates with PAJAIS column)
+
+TOOL 6: generate_comparison_csv()
+  - WHEN: Phase 6 — only after BOTH abstract AND title runs have taxonomy_map.json
+  - WHAT: Creates side-by-side comparison of abstract vs title themes
+  - OUTPUT: comparison.csv
+
+TOOL 7: export_narrative(run_key)
+  - WHEN: Phase 6 — after researcher confirms comparison.csv via Submit Review
+  - WHAT: Generates 500-word Section 7 for conference paper via Mistral
+  - OUTPUT: narrative.txt
+
+═══════════════════════════════════════════════════════════════════════════
+BRAUN & CLARKE (2006) SIX-PHASE THEMATIC ANALYSIS — FULL WORKFLOW
+═══════════════════════════════════════════════════════════════════════════
+
+─────────────────────────────────────────────────────────────────────────
+PHASE 1 — FAMILIARISATION WITH THE DATA
+─────────────────────────────────────────────────────────────────────────
+TRIGGER: CSV uploaded or researcher says "analyze CSV" or "start" or "load data"
+
+ACTIONS:
+  1. Call load_scopus_csv(filepath) with the uploaded file path.
+  2. Display the returned statistics clearly.
+  3. Explain: "Familiarisation involves reading and re-reading the data to understand its scope
+     and content before any coding begins (Braun & Clarke, 2006)."
+  4. Ask researcher to type "run abstract" to begin Phase 2 on abstracts.
+
+RESPONSE FORMAT:
+  - Show paper count, sentence counts, year range
+  - Briefly explain what BERTopic will do in Phase 2
+  - End with: "Type **'run abstract'** when ready."
+
+★ STOP GATE 1 ★
+STOP HERE AFTER PHASE 1. Do NOT call any other tool.
+Wait for researcher to type "run abstract" or "run title".
+
+─────────────────────────────────────────────────────────────────────────
+PHASE 2 — GENERATING INITIAL CODES
+─────────────────────────────────────────────────────────────────────────
+TRIGGER: Researcher types "run abstract" or "run title"
+
+ACTIONS:
+  1. Call run_bertopic_discovery(run_key="abstract", threshold=0.7)
+     [or run_key="title" if researcher specified "run title"]
+  2. Immediately after (in same message), call label_topics_with_llm(run_key=...)
+  3. Tell researcher: The review table now shows all labeled topics.
+  4. Instruct researcher how to use the table:
+     - APPROVE column: Enter "yes" to keep, "no" to reject, "merge:X" to merge with topic X
+     - RENAME TO column: Enter new name if desired
+     - REASONING column: Brief justification for decision
+  5. Explain: "Initial coding systematically labels features of the data relevant to the
+     research question (Braun & Clarke, 2006, p. 88)."
+
+RESPONSE FORMAT:
+  - Confirm topics discovered and sentences clustered
+  - Show top 5 topics as examples with their labels and sentence counts
+  - Explain what threshold=0.7 means (produces ~100 fine-grained topics)
+  - End with: "**Review the table below. Edit Approve/Rename/Reasoning columns, then click Submit Review.**"
+
+★ STOP GATE 2 (MANDATORY) ★
+STOP HERE AFTER PHASE 2. Do NOT proceed to Phase 3 automatically.
+Do NOT consolidate themes. Do NOT call any other tool.
+WAIT for researcher to click Submit Review and send the review table data.
+
+─────────────────────────────────────────────────────────────────────────
+PHASE 3 — SEARCHING FOR THEMES
+─────────────────────────────────────────────────────────────────────────
+TRIGGER: Researcher submits review table (table data appears in message)
+
+ACTIONS:
+  1. Parse the researcher's review table decisions from the message.
+  2. Build theme_map from approved topics: group topics with same RENAME TO into themes.
+     Example: If topics 0, 1, 5 all have RENAME TO = "AI Tourism", group them.
+  3. Call consolidate_into_themes(run_key=..., theme_map='{"AI Tourism": [0,1,5], ...}')
+  4. Display the consolidated themes with their sentence counts.
+  5. Explain: "Searching for themes involves collating codes into potential themes and gathering
+     relevant coded data (Braun & Clarke, 2006, p. 89)."
+
+RESPONSE FORMAT:
+  - List each consolidated theme: name, topics merged, sentence count
+  - Note any rejected topics (Approve=no) that were excluded
+  - End with: "**Review the consolidated themes in the table. Click Submit Review to proceed to Phase 4.**"
+
+★ STOP GATE 3 (MANDATORY) ★
+STOP HERE AFTER PHASE 3. Do NOT proceed to Phase 4 automatically.
+Wait for researcher to click Submit Review again.
+
+─────────────────────────────────────────────────────────────────────────
+PHASE 4 — REVIEWING THEMES (SATURATION CHECK)
+─────────────────────────────────────────────────────────────────────────
+TRIGGER: Researcher submits review table after Phase 3
+
+ACTIONS:
+  1. Review the themes from themes.json.
+  2. Check for saturation: Do themes adequately cover the data? Are there overlapping themes?
+     Are any themes too broad or too narrow?
+  3. Report saturation status based on:
+     - Coverage: What % of sentences are captured by themes?
+     - Coherence: Do themes have internal consistency?
+     - Distinctiveness: Are themes sufficiently different from each other?
+  4. Recommend any merges or splits if needed.
+  5. Explain: "Reviewing themes ensures themes work in relation to the coded extracts and
+     the entire dataset (Braun & Clarke, 2006, p. 91)."
+
+RESPONSE FORMAT:
+  - Report: X themes covering Y sentences (Z% of total)
+  - Saturation assessment: ACHIEVED / NEEDS REVISION
+  - Specific recommendations if revision needed
+  - End with: "**Confirm or adjust themes in the table. Click Submit Review to proceed to Phase 5.**"
+
+★ STOP GATE 4 (MANDATORY) ★
+STOP HERE AFTER PHASE 4. Do NOT proceed to Phase 5 automatically.
+Wait for researcher to click Submit Review.
+
+─────────────────────────────────────────────────────────────────────────
+PHASE 5 — DEFINING AND NAMING THEMES
+─────────────────────────────────────────────────────────────────────────
+TRIGGER: Researcher submits review table after Phase 4
+
+ACTIONS:
+  1. Present final theme names and definitions.
+  2. For each theme, provide:
+     - Concise name (3-5 words)
+     - One-sentence definition capturing the essence
+     - Key evidence sentences (from top_sentences)
+  3. Invite researcher to finalise names via the RENAME TO column.
+  4. Explain: "Defining and naming themes involves identifying the 'essence' of each theme
+     and determining the aspect of the data each theme captures (Braun & Clarke, 2006, p. 92)."
+
+RESPONSE FORMAT:
+  - List each theme with proposed name and definition
+  - Show 2 evidence sentences per theme
+  - End with: "**Edit Rename To column if needed. Click Submit Review to proceed to Phase 5.5 (PAJAIS mapping).**"
+
+─────────────────────────────────────────────────────────────────────────
+PHASE 5.5 — PAJAIS TAXONOMY MAPPING
+─────────────────────────────────────────────────────────────────────────
+TRIGGER: Researcher submits review table after Phase 5
+
+ACTIONS:
+  1. Call compare_with_taxonomy(run_key=...)
+  2. The review table's "Top Evidence" column now shows:
+     "→ PAJAIS: [Category Name] | Confidence: X.XX | [reasoning]" for MAPPED themes
+     "→ NOVEL | [reason why no category fits]" for NOVEL themes
+  3. Highlight NOVEL themes as potential research contributions.
+  4. Explain the PAJAIS taxonomy and what NOVEL means for publications.
+
+RESPONSE FORMAT:
+  - Summary: X MAPPED, Y NOVEL themes
+  - List NOVEL themes explicitly — these are research gaps
+  - End with: "**Review PAJAIS mapping in the table. NOVEL themes = publishable research gaps.
+    Click Submit Review to proceed to Phase 6 (Report Generation).**"
+
+★ STOP GATE 5 (MANDATORY) ★
+STOP HERE AFTER PHASE 5.5. Do NOT proceed to Phase 6 automatically.
+Wait for researcher to click Submit Review.
+
+─────────────────────────────────────────────────────────────────────────
+PHASE 6 — PRODUCING THE REPORT
+─────────────────────────────────────────────────────────────────────────
+TRIGGER: Researcher submits review table after Phase 5.5
+
+ACTIONS:
+  Step 6a — Comparison CSV:
+    1. Check if BOTH abstract and title taxonomy_map.json files exist.
+    2. If both exist: Call generate_comparison_csv()
+    3. If only one run complete: Inform researcher which run is missing.
+    4. End with: "**Check Download tab for comparison.csv. Click Submit Review to generate narrative.**"
+
+  Step 6b — Narrative (after researcher confirms):
+    5. Call export_narrative(run_key=...) for the current run.
+    6. Congratulate researcher on completing the analysis.
+    7. List all downloadable files in the Download tab.
+
+RESPONSE FORMAT:
+  - Confirm comparison.csv is ready (if both runs complete)
+  - Confirm narrative.txt is generated
+  - List all output files: comparison.csv, abstract_taxonomy_map.json,
+    title_taxonomy_map.json, abstract_narrative.txt, title_narrative.txt
+  - End with: "**Download all files from the Download tab for your conference paper Section 7.**"
+
+═══════════════════════════════════════════════════════════════════════════
+STOP GATE SUMMARY (4 Mandatory Gates)
+═══════════════════════════════════════════════════════════════════════════
+Gate 1 → After Phase 1 (Load): Wait for "run abstract" or "run title"
+Gate 2 → After Phase 2 (Codes): Wait for Submit Review (researcher approves topics)
+Gate 3 → After Phase 3 (Themes): Wait for Submit Review (researcher confirms themes)
+Gate 4 → After Phase 4 (Saturation): Wait for Submit Review (researcher confirms saturation)
+Gate 5 → After Phase 5.5 (PAJAIS): Wait for Submit Review (researcher reviews taxonomy)
+
+ALL FIVE GATES ARE MANDATORY. Skipping any gate violates the researcher-in-the-loop principle.
+
+═══════════════════════════════════════════════════════════════════════════
+ERROR HANDLING GUIDANCE
+═══════════════════════════════════════════════════════════════════════════
+If a tool returns an error:
+1. Read the error message carefully.
+2. Diagnose the likely cause (missing file, wrong key, API issue).
+3. Explain the error to the researcher in plain language.
+4. Suggest a corrective action (e.g., re-upload CSV, retry, check API key).
+5. Do NOT crash. Do NOT give up. Adapt strategy.
+
+If theme_map parsing fails:
+- Ask researcher to re-submit the review table clearly.
+- Provide an example of valid approve/rename instructions.
+
+═══════════════════════════════════════════════════════════════════════════
+TONE AND COMMUNICATION STYLE
+═══════════════════════════════════════════════════════════════════════════
+- Professional yet approachable
+- Reference Braun & Clarke (2006) when explaining phases
+- Use clear section headers in responses (Phase X — Name)
+- Use emojis sparingly for visual cues (✅ ⬜ 🔢 📊 🏷️)
+- Always end with a clear call-to-action for the researcher
+- Never use jargon without explanation
+"""
+
+# ── Agent Factory ──────────────────────────────────────────────────────────────
+
+def create_agent():
+    """Create and return the LangGraph ReAct agent with Mistral LLM and MemorySaver."""
+    llm = ChatMistralAI(
+        model="mistral-small-latest",
+        api_key=os.environ.get("MISTRAL_API_KEY", ""),
+        temperature=0.1,
+    )
+    memory = MemorySaver()
+    agent = create_react_agent(
+        llm,
+        ALL_TOOLS,
+        prompt=SYSTEM_PROMPT,
+        checkpointer=memory,
+    )
+    return agent
+
+
+# Singleton agent instance
+_agent = None
+
+
+def get_agent():
+    """Return singleton agent instance (created once on first call)."""
+    global _agent
+    _agent = _agent or create_agent()
+    return _agent
+
+
+def invoke_agent(message: str, thread_id: str = "default") -> str:
+    """Invoke the agent with a user message and return its response text.
+    thread_id: conversation thread identifier for memory isolation."""
+    agent = get_agent()
+    config = {"configurable": {"thread_id": thread_id}}
+    result = agent.invoke({"messages": [("user", message)]}, config=config)
+    messages = result.get("messages", [])
+    last = messages[-1] if messages else None
+    return last.content if last and hasattr(last, "content") else str(last)

app.py

@@ -0,0 +1,420 @@
+"""
+app.py — Gradio UI for BERTopic Agentic AI Application (~370 lines)
+Sections: ① Data Input  ② Agent Conversation  ③ Results (Table | Charts | Download)
+Rules: ZERO business logic here. All decisions made by agent.py.
+"""
+
+import os
+import json
+import glob
+import gradio as gr
+from agent import invoke_agent
+
+CHECKPOINT_DIR = "checkpoints"
+os.makedirs(CHECKPOINT_DIR, exist_ok=True)
+
+CSV_PATH = os.path.join(CHECKPOINT_DIR, "uploaded.csv")
+
+# ── Checkpoint file paths ──────────────────────────────────────────────────────
+def ckpt(name):
+    return os.path.join(CHECKPOINT_DIR, name)
+
+
+# ── Phase progress HTML ────────────────────────────────────────────────────────
+def build_phase_bar():
+    phases = [
+        ("① Load", "stats.json"),
+        ("② Codes", "abstract_labels.json"),
+        ("③ Themes", "abstract_themes.json"),
+        ("④ Saturation", "abstract_themes.json"),
+        ("⑤ Names", "abstract_themes.json"),
+        ("⑤½ PAJAIS", "abstract_taxonomy_map.json"),
+        ("⑥ Report", "comparison.csv"),
+    ]
+    items = list(map(
+        lambda p: (
+            f'<div style="display:inline-flex;align-items:center;gap:6px;'
+            f'padding:6px 14px;border-radius:20px;font-size:13px;font-weight:600;'
+            f'background:{"#22c55e" if os.path.exists(ckpt(p[1])) else "#374151"};'
+            f'color:{"#fff" if os.path.exists(ckpt(p[1])) else "#9ca3af"};">'
+            f'{"✅" if os.path.exists(ckpt(p[1])) else "⬜"} {p[0]}</div>'
+        ),
+        phases,
+    ))
+    bar = (
+        '<div style="background:#111827;padding:12px 16px;border-radius:12px;'
+        'border:1px solid #1f2937;display:flex;flex-wrap:wrap;gap:8px;align-items:center;">'
+        '<span style="color:#6b7280;font-size:12px;font-weight:700;margin-right:4px;">B&amp;C PHASES:</span>'
+        + "".join(items)
+        + "</div>"
+    )
+    return bar
+
+
+# ── Review table loading ───────────────────────────────────────────────────────
+def load_review_table():
+    """Priority: taxonomy_map → themes → labels → summaries"""
+    priority = [
+        ("abstract_taxonomy_map.json", "taxonomy"),
+        ("abstract_themes.json", "themes"),
+        ("abstract_labels.json", "labels"),
+        ("abstract_summaries.json", "summaries"),
+    ]
+    for filename, mode in priority:
+        path = ckpt(filename)
+        if os.path.exists(path):
+            with open(path) as f:
+                data = json.load(f)
+            return _format_table(data, mode)
+    return _empty_table()
+
+
+def _empty_table():
+    return [[None] * 8]
+
+
+def _format_table(data, mode):
+    rows = list(map(lambda item: _format_row(item, mode), data))
+    return rows if rows else _empty_table()
+
+
+def _format_row(item, mode):
+    idx = item.get("topic_id", item.get("name", ""))
+    label = item.get("label", item.get("name", ""))
+
+    if mode == "taxonomy":
+        evidence = (
+            f"→ {item.get('pajais_match', 'NOVEL')} "
+            f"| conf: {item.get('match_confidence', 0):.2f} "
+            f"| {item.get('reasoning', '')}"
+        )
+    else:
+        sentences = item.get("top_sentences", [])
+        evidence = sentences[0] if sentences else ""
+
+    sentences_count = item.get("sentence_count", len(item.get("top_sentences", [])))
+    papers = item.get("paper_count", "")
+    approve = item.get("approve", "yes")
+    rename = item.get("rename_to", label)
+    reasoning = item.get("reasoning", "")
+
+    return [idx, label, evidence, sentences_count, papers, approve, rename, reasoning]
+
+
+# ── Chart list ────────────────────────────────────────────────────────────────
+def get_chart_choices():
+    chart_files = glob.glob(ckpt("*_chart_*.html"))
+    choices = list(map(
+        lambda f: os.path.basename(f).replace("_", " ").replace(".html", "").title(),
+        chart_files,
+    ))
+    return choices if choices else ["No charts yet"]
+
+
+def load_chart_html(choice):
+    if not choice or choice == "No charts yet":
+        return "<p style='color:#6b7280;padding:20px;'>Charts appear after Phase 2 analysis.</p>"
+    filename = choice.lower().replace(" ", "_") + ".html"
+    path = ckpt(filename)
+    if os.path.exists(path):
+        with open(path) as f:
+            content = f.read()
+        return f'<iframe srcdoc="{content.replace(chr(34), "&quot;")}" width="100%" height="600px" frameborder="0"></iframe>'
+    return "<p style='color:#ef4444;'>Chart file not found.</p>"
+
+
+# ── Download file list ─────────────────────��───────────────────────────────────
+def get_download_files():
+    patterns = [
+        "*.csv", "*.json", "*.txt", "*.npy",
+    ]
+    files = []
+    list(map(lambda p: files.extend(glob.glob(ckpt(p))), patterns))
+    files.sort(key=os.path.getmtime, reverse=True)
+    return files if files else None
+
+
+# ── Table-to-theme-map parser ──────────────────────────────────────────────────
+def parse_table_to_message(table_data):
+    """Convert review table edits into a structured message for the agent."""
+    if not table_data:
+        return "Submit Review: No table data provided."
+
+    approved = list(filter(lambda row: len(row) >= 6 and str(row[5]).lower() in ("yes", "y", "1", "true"), table_data))
+    rejected = list(filter(lambda row: len(row) >= 6 and str(row[5]).lower() in ("no", "n", "0", "false"), table_data))
+
+    theme_groups = {}
+    list(map(
+        lambda row: theme_groups.setdefault(str(row[6]) if len(row) > 6 and row[6] else str(row[1]), []).append(int(row[0]) if str(row[0]).isdigit() else row[0]),
+        approved,
+    ))
+
+    theme_map_str = json.dumps(theme_groups)
+
+    msg = (
+        f"Submit Review received.\n\n"
+        f"Approved topics: {len(approved)}\n"
+        f"Rejected topics: {len(rejected)}\n\n"
+        f"Theme groupings (RENAME TO → [topic_ids]):\n{theme_map_str}\n\n"
+        f"Researcher reasoning summary:\n"
+        + "\n".join(list(map(
+            lambda row: f"  - Topic {row[0]} ({row[1]}): {row[7]}" if len(row) > 7 and row[7] else "",
+            approved,
+        )))
+        + "\n\nPlease proceed to the next phase based on these decisions."
+    )
+    return msg
+
+
+# ── Main Gradio App ────────────────────────────────────────────────────────────
+def build_app():
+    with gr.Blocks(
+        title="BERTopic Thematic Analysis Agent",
+        theme=gr.themes.Base(
+            primary_hue="emerald",
+            secondary_hue="slate",
+            neutral_hue="slate",
+            font=[gr.themes.GoogleFont("IBM Plex Mono"), "monospace"],
+        ),
+        css="""
+        body { background: #0a0f1a !important; }
+        .gradio-container { max-width: 1400px !important; background: #0a0f1a !important; }
+        .section-box {
+            border: 1px solid #1e293b;
+            border-radius: 16px;
+            padding: 20px;
+            background: #0f172a;
+            margin-bottom: 16px;
+        }
+        .section-header {
+            font-size: 13px;
+            font-weight: 700;
+            color: #64748b;
+            letter-spacing: 0.12em;
+            text-transform: uppercase;
+            margin-bottom: 12px;
+            padding-bottom: 8px;
+            border-bottom: 1px solid #1e293b;
+        }
+        .gr-button-primary {
+            background: #10b981 !important;
+            border: none !important;
+            font-weight: 700 !important;
+        }
+        .gr-button-secondary {
+            background: #1e293b !important;
+            border: 1px solid #334155 !important;
+            color: #94a3b8 !important;
+        }
+        footer { display: none !important; }
+        """,
+    ) as app:
+
+        # ── Header ──────────────────────────────────────────────────────────
+        gr.HTML("""
+        <div style="text-align:center;padding:32px 0 16px;background:linear-gradient(180deg,#0f172a 0%,#0a0f1a 100%);">
+            <div style="font-family:'IBM Plex Mono',monospace;font-size:11px;letter-spacing:0.3em;
+                        color:#10b981;text-transform:uppercase;margin-bottom:8px;">
+                Braun &amp; Clarke (2006) · BERTopic · PAJAIS Taxonomy
+            </div>
+            <h1 style="font-family:'IBM Plex Mono',monospace;font-size:28px;font-weight:700;
+                       color:#f1f5f9;margin:0 0 8px;">
+                Thematic Analysis Agent
+            </h1>
+            <p style="color:#475569;font-size:14px;margin:0;">
+                Agentic AI · LangGraph · Mistral LLM · AgglomerativeClustering (cosine, 384d)
+            </p>
+        </div>
+        """)
+
+        # Phase progress bar
+        phase_bar = gr.HTML(value=build_phase_bar(), label="Phase Progress")
+
+        # ── SECTION 1: Data Input ────────────────────────────────────────────
+        gr.HTML('<div class="section-header">① DATA INPUT</div>')
+        with gr.Row():
+            csv_upload = gr.File(
+                label="Upload Scopus CSV Export",
+                file_types=[".csv"],
+                scale=2,
+            )
+            with gr.Column(scale=1):
+                gr.HTML("""
+                <div style="background:#1e293b;border-radius:12px;padding:16px;font-size:13px;color:#94a3b8;">
+                    <b style="color:#f1f5f9;">Required CSV Columns:</b><br>
+                    Authors · Title · Abstract<br>
+                    Author Keywords · Cited by<br>
+                    Source title · Year
+                </div>
+                """)
+
+        # ── SECTION 2: Agent Conversation ───────────────────────────────────
+        gr.HTML('<div class="section-header">② AGENT CONVERSATION</div>')
+        chatbot = gr.Chatbot(
+            label="Thematic Analysis Agent",
+            height=500,
+            bubble_full_width=False,
+            show_copy_button=True,
+            render_markdown=True,
+            avatar_images=(None, "https://www.anthropic.com/favicon.ico"),
+            type="messages",
+        )
+        with gr.Row():
+            user_input = gr.Textbox(
+                placeholder="Type 'run abstract', 'run title', or any instruction...",
+                label="",
+                scale=5,
+                lines=1,
+                container=False,
+            )
+            send_btn = gr.Button("Send ▶", variant="primary", scale=1)
+
+        # ── SECTION 3: Results ───────────────────────────────────────────────
+        gr.HTML('<div class="section-header">③ RESULTS</div>')
+        with gr.Tabs():
+
+            # Tab 1: Review Table
+            with gr.TabItem("📋 Review Table"):
+                gr.HTML("""
+                <p style="color:#94a3b8;font-size:13px;margin-bottom:8px;">
+                    Edit <b>Approve</b> (yes/no), <b>Rename To</b>, and <b>Reasoning</b> columns.
+                    Then click <b>Submit Review</b> to send decisions to the agent.
+                </p>
+                """)
+                review_table = gr.Dataframe(
+                    headers=["#", "Topic Label", "Top Evidence", "Sentences", "Papers", "Approve", "Rename To", "Reasoning"],
+                    datatype=["str", "str", "str", "number", "str", "str", "str", "str"],
+                    row_count=(10, "dynamic"),
+                    col_count=(8, "fixed"),
+                    interactive=True,
+                    wrap=True,
+                    label="",
+                )
+                submit_review_btn = gr.Button("📤 Submit Review →", variant="primary")
+
+            # Tab 2: Charts
+            with gr.TabItem("📊 Charts"):
+                chart_dropdown = gr.Dropdown(
+                    choices=get_chart_choices(),
+                    label="Select Chart",
+                    interactive=True,
+                )
+                refresh_charts_btn = gr.Button("🔄 Refresh Chart List", variant="secondary", size="sm")
+                chart_display = gr.HTML(
+                    value="<p style='color:#6b7280;padding:20px;'>Charts appear after Phase 2 BERTopic analysis.</p>"
+                )
+
+            # Tab 3: Downloads
+            with gr.TabItem("📥 Download Files"):
+                gr.HTML("""
+                <p style="color:#94a3b8;font-size:13px;margin-bottom:8px;">
+                    All checkpoint files are listed below. Download for your conference paper.
+                </p>
+                """)
+                download_files = gr.File(
+                    label="Output Files",
+                    file_count="multiple",
+                    interactive=False,
+                )
+                refresh_downloads_btn = gr.Button("🔄 Refresh Files", variant="secondary", size="sm")
+
+        # ── State ─────────────────────────────────────────────────────────────
+        thread_state = gr.State("default")
+
+        # ── Event: CSV Upload ─────────────────────────────────────────────────
+        def on_csv_upload(file, history, thread_id):
+            if file is None:
+                return history, build_phase_bar(), load_review_table()
+            msg = f"Analyze my Scopus CSV: {file.name}"
+            history = history or []
+            history.append({"role": "user", "content": msg})
+            response = invoke_agent(f"load_scopus_csv filepath={file.name}", thread_id)
+            history.append({"role": "assistant", "content": response})
+            return history, build_phase_bar(), load_review_table()
+
+        csv_upload.upload(
+            on_csv_upload,
+            inputs=[csv_upload, chatbot, thread_state],
+            outputs=[chatbot, phase_bar, review_table],
+        )
+
+        # ── Event: Send message ───────────────────────────────────────────────
+        def on_send(message, history, thread_id):
+            if not message.strip():
+                return history, "", build_phase_bar(), load_review_table()
+            history = history or []
+            history.append({"role": "user", "content": message})
+            response = invoke_agent(message, thread_id)
+            history.append({"role": "assistant", "content": response})
+            return history, "", build_phase_bar(), load_review_table()
+
+        send_btn.click(
+            on_send,
+            inputs=[user_input, chatbot, thread_state],
+            outputs=[chatbot, user_input, phase_bar, review_table],
+        )
+        user_input.submit(
+            on_send,
+            inputs=[user_input, chatbot, thread_state],
+            outputs=[chatbot, user_input, phase_bar, review_table],
+        )
+
+        # ── Event: Submit Review ──────────────────────────────────────────────
+        def on_submit_review(table_data, history, thread_id):
+            msg = parse_table_to_message(table_data)
+            history = history or []
+            history.append({"role": "user", "content": "📤 Submit Review (table decisions sent to agent)"})
+            response = invoke_agent(msg, thread_id)
+            history.append({"role": "assistant", "content": response})
+            return history, build_phase_bar(), load_review_table()
+
+        submit_review_btn.click(
+            on_submit_review,
+            inputs=[review_table, chatbot, thread_state],
+            outputs=[chatbot, phase_bar, review_table],
+        )
+
+        # ── Event: Chart selection ────────────────────────────────────────────
+        chart_dropdown.change(
+            load_chart_html,
+            inputs=[chart_dropdown],
+            outputs=[chart_display],
+        )
+
+        def refresh_charts():
+            choices = get_chart_choices()
+            return gr.update(choices=choices, value=choices[0] if choices else None)
+
+        refresh_charts_btn.click(
+            refresh_charts,
+            outputs=[chart_dropdown],
+        )
+
+        # ── Event: Download refresh ───────────────────────────────────────────
+        def refresh_downloads():
+            files = get_download_files()
+            return gr.update(value=files)
+
+        refresh_downloads_btn.click(
+            refresh_downloads,
+            outputs=[download_files],
+        )
+
+        # ── Initial load ──────────────────────────────────────────────────────
+        app.load(
+            lambda: (build_phase_bar(), load_review_table(), get_download_files()),
+            outputs=[phase_bar, review_table, download_files],
+        )
+
+    return app
+
+
+# ── Launch ─────────────────────────────────────────────────────────────────────
+if __name__ == "__main__":
+    demo = build_app()
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        ssr_mode=False,
+        share=False,
+    )

requirements.txt

@@ -0,0 +1,13 @@
+gradio
+langchain-core
+langchain-mistralai
+langgraph
+sentence-transformers
+scikit-learn
+bertopic
+plotly
+numpy
+pandas
+hdbscan
+umap-learn
+pynndescent

tools.py

@@ -0,0 +1,571 @@
+"""
+tools.py — 7 @tool functions for BERTopic Agentic AI Application
+Rules: ZERO if/else, ZERO for/while, ZERO try/except. All decisions by LLM.
+"""
+
+import os
+import json
+import re
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+import plotly.express as px
+from plotly.subplots import make_subplots
+from langchain_core.tools import tool
+from sentence_transformers import SentenceTransformer
+from sklearn.cluster import AgglomerativeClustering
+from sklearn.metrics.pairwise import cosine_similarity
+from langchain_core.prompts import PromptTemplate
+from langchain_core.output_parsers import JsonOutputParser
+from langchain_mistralai import ChatMistralAI
+
+# ── Constants ──────────────────────────────────────────────────────────────────
+NEAREST_K = 5
+MAX_LABEL_TOPICS = 100
+CHECKPOINT_DIR = "checkpoints"
+os.makedirs(CHECKPOINT_DIR, exist_ok=True)
+
+RUN_CONFIGS = {
+    "abstract": ["Abstract"],
+    "title": ["Title"],
+}
+
+BOILERPLATE_PATTERNS = [
+    r"©\s*\d{4}.*",
+    r"All rights reserved.*",
+    r"Published by Elsevier.*",
+    r"doi:.*",
+    r"http[s]?://\S+",
+    r"www\.\S+",
+    r"This article is.*",
+    r"Please cite.*",
+    r"Correspondence to.*",
+    r"E-mail address.*",
+    r"Received \d+.*",
+    r"Accepted \d+.*",
+    r"Available online.*",
+    r"Keywords:.*",
+    r"Abstract\.?\s*$",
+    r"^\s*\d+\s*$",
+    r"Springer.*",
+    r"Taylor & Francis.*",
+    r"Wiley.*",
+    r"IEEE.*",
+    r"ACM.*",
+    r"Sage Publications.*",
+]
+
+PAJAIS_CATEGORIES = [
+    "1. Smart Tourism Technologies",
+    "2. AI and Machine Learning in Tourism",
+    "3. Big Data Analytics in Hospitality",
+    "4. Social Media and User-Generated Content",
+    "5. Mobile Technologies and Applications",
+    "6. Blockchain in Travel and Tourism",
+    "7. Internet of Things in Hospitality",
+    "8. Robotics and Automation",
+    "9. Augmented and Virtual Reality",
+    "10. Revenue Management and Pricing",
+    "11. Customer Experience and Satisfaction",
+    "12. Online Reviews and Reputation Management",
+    "13. Digital Marketing and e-Commerce",
+    "14. Sharing Economy Platforms",
+    "15. Destination Management Systems",
+    "16. Sustainable and Green Technologies",
+    "17. Crisis Management and Resilience",
+    "18. Human-Computer Interaction",
+    "19. Recommendation Systems",
+    "20. Natural Language Processing in Tourism",
+    "21. Computer Vision in Hospitality",
+    "22. Cybersecurity and Privacy",
+    "23. Supply Chain and Logistics",
+    "24. Accessibility and Inclusive Technology",
+    "25. Metaverse and Immersive Experiences",
+]
+
+CSV_PATH = os.path.join(CHECKPOINT_DIR, "uploaded.csv")
+
+
+def _ckpt(name):
+    return os.path.join(CHECKPOINT_DIR, name)
+
+
+def _llm():
+    return ChatMistralAI(
+        model="mistral-small-latest",
+        api_key=os.environ.get("MISTRAL_API_KEY", ""),
+        temperature=0.1,
+    )
+
+
+def _clean_sentence(s):
+    cleaned = s.strip()
+    cleaned = re.sub("|".join(BOILERPLATE_PATTERNS), "", cleaned, flags=re.IGNORECASE)
+    return cleaned.strip()
+
+
+def _split_sentences(text):
+    from nltk.tokenize import sent_tokenize
+    import nltk
+    nltk.download("punkt", quiet=True)
+    nltk.download("punkt_tab", quiet=True)
+    sentences = sent_tokenize(str(text))
+    cleaned = list(map(_clean_sentence, sentences))
+    return list(filter(lambda s: len(s) > 20, cleaned))
+
+
+# ── Tool 1: load_scopus_csv ────────────────────────────────────────────────────
+@tool(handle_tool_error=True)
+def load_scopus_csv(filepath: str) -> str:
+    """Load a Scopus CSV export, count papers and sentences, apply boilerplate filtering.
+    Returns stats string with paper count, abstract sentence count, title sentence count.
+    filepath: path to the uploaded CSV file."""
+    df = pd.read_csv(filepath, encoding="utf-8-8-sig")
+    df.to_csv(CSV_PATH, index=False)
+
+    paper_count = len(df)
+    abstract_sentences = list(
+        filter(None, sum(map(_split_sentences, df["Abstract"].dropna().tolist()), []))
+    )
+    title_sentences = list(
+        filter(None, sum(map(_split_sentences, df["Title"].dropna().tolist()), []))
+    )
+
+    stats = {
+        "papers": paper_count,
+        "abstract_sentences": len(abstract_sentences),
+        "title_sentences": len(title_sentences),
+        "columns": list(df.columns),
+        "year_range": f"{int(df['Year'].min())} – {int(df['Year'].max())}" if "Year" in df.columns else "N/A",
+    }
+    with open(_ckpt("stats.json"), "w") as f:
+        json.dump(stats, f, indent=2)
+
+    return (
+        f"✅ CSV loaded successfully.\n"
+        f"📄 Papers: {paper_count}\n"
+        f"📝 Abstract sentences (after cleaning): {len(abstract_sentences)}\n"
+        f"🔤 Title sentences (after cleaning): {len(title_sentences)}\n"
+        f"📅 Year range: {stats['year_range']}\n"
+        f"📊 Columns: {', '.join(stats['columns'])}\n\n"
+        f"Data is ready. Please type **'run abstract'** to begin Phase 2 BERTopic analysis on abstracts."
+    )
+
+
+# ── Tool 2: run_bertopic_discovery ────────────────────────────────────────────
+@tool(handle_tool_error=True)
+def run_bertopic_discovery(run_key: str, threshold: float = 0.7) -> str:
+    """Embed sentences with all-MiniLM-L6-v2, cluster with AgglomerativeClustering (cosine metric),
+    find 5 nearest centroids per cluster, generate 4 Plotly charts. Save summaries.json + emb.npy.
+    run_key: 'abstract' or 'title'. threshold: clustering distance threshold (default 0.7)."""
+    df = pd.read_csv(CSV_PATH, encoding="utf-8-8-sig")
+    columns = RUN_CONFIGS[run_key]
+
+    texts = sum(
+        list(map(lambda col: df[col].dropna().tolist(), columns)), []
+    )
+    sentences = list(
+        filter(lambda s: len(s) > 20,
+               sum(list(map(_split_sentences, texts)), []))
+    )
+
+    model = SentenceTransformer("all-MiniLM-L6-v2")
+    embeddings = model.encode(sentences, normalize_embeddings=True, show_progress_bar=False)
+    np.save(_ckpt(f"{run_key}_emb.npy"), embeddings)
+
+    clustering = AgglomerativeClustering(
+        metric="cosine",
+        linkage="average",
+        distance_threshold=threshold,
+        n_clusters=None,
+    )
+    labels_arr = clustering.fit_predict(embeddings)
+
+    unique_labels = list(set(labels_arr.tolist()))
+    cluster_data = list(map(lambda lbl: _build_cluster_summary(lbl, labels_arr, sentences, embeddings), unique_labels))
+    cluster_data.sort(key=lambda x: x["sentence_count"], reverse=True)
+
+    with open(_ckpt(f"{run_key}_summaries.json"), "w") as f:
+        json.dump(cluster_data, f, indent=2)
+
+    _generate_charts(cluster_data, run_key, embeddings, labels_arr)
+
+    return (
+        f"✅ BERTopic discovery complete for **{run_key}** run.\n"
+        f"🔢 Topics discovered: {len(unique_labels)}\n"
+        f"📊 Sentences clustered: {len(sentences)}\n"
+        f"📁 Saved: {run_key}_summaries.json, {run_key}_emb.npy\n"
+        f"🎨 4 Plotly charts generated.\n\n"
+        f"Now calling label_topics_with_llm to label the top {MAX_LABEL_TOPICS} topics..."
+    )
+
+
+def _build_cluster_summary(lbl, labels_arr, sentences, embeddings):
+    mask = np.array(labels_arr) == lbl
+    cluster_sents = [s for s, m in zip(sentences, mask.tolist()) if m]
+    cluster_embs = embeddings[mask]
+    centroid = cluster_embs.mean(axis=0, keepdims=True)
+    sims = cosine_similarity(centroid, cluster_embs)[0]
+    top_idxs = np.argsort(sims)[::-1][:NEAREST_K].tolist()
+    top_sents = [cluster_sents[i] for i in top_idxs]
+    return {
+        "topic_id": int(lbl),
+        "sentence_count": len(cluster_sents),
+        "top_sentences": top_sents,
+        "centroid": centroid[0].tolist(),
+        "label": f"Topic_{lbl}",
+        "category": "",
+        "confidence": 0.0,
+        "reasoning": "",
+        "niche": False,
+    }
+
+
+def _generate_charts(cluster_data, run_key, embeddings, labels_arr):
+    top_n = min(30, len(cluster_data))
+    top_clusters = cluster_data[:top_n]
+    topic_ids = list(map(lambda c: c["topic_id"], top_clusters))
+    counts = list(map(lambda c: c["sentence_count"], top_clusters))
+    topic_labels = list(map(lambda c: c["label"], top_clusters))
+
+    # Chart 1: Bar chart — top topics by sentence count
+    fig_bar = px.bar(
+        x=counts, y=topic_labels, orientation="h",
+        title=f"Top {top_n} Topics by Sentence Count ({run_key})",
+        labels={"x": "Sentences", "y": "Topic"},
+        color=counts, color_continuous_scale="Viridis",
+    )
+    fig_bar.update_layout(height=700, yaxis=dict(autorange="reversed"))
+    with open(_ckpt(f"{run_key}_chart_bar.html"), "w") as f:
+        f.write(fig_bar.to_html(include_plotlyjs="cdn", full_html=True))
+
+    # Chart 2: Intertopic map (2D PCA projection of centroids)
+    centroids = np.array(list(map(lambda c: c["centroid"], top_clusters)))
+    from sklearn.decomposition import PCA
+    pca = PCA(n_components=2)
+    coords = pca.fit_transform(centroids)
+    fig_map = px.scatter(
+        x=coords[:, 0], y=coords[:, 1],
+        text=topic_labels, size=counts,
+        title=f"Intertopic Distance Map ({run_key})",
+        labels={"x": "PC1", "y": "PC2"},
+        color=counts, color_continuous_scale="Plasma",
+    )
+    fig_map.update_traces(textposition="top center")
+    fig_map.update_layout(height=600)
+    with open(_ckpt(f"{run_key}_chart_map.html"), "w") as f:
+        f.write(fig_map.to_html(include_plotlyjs="cdn", full_html=True))
+
+    # Chart 3: Hierarchy (dendrogram-style using sorted counts)
+    sorted_data = sorted(zip(topic_labels, counts), key=lambda x: x[1])
+    fig_hier = go.Figure(go.Bar(
+        x=list(map(lambda d: d[1], sorted_data)),
+        y=list(map(lambda d: d[0], sorted_data)),
+        orientation="h",
+        marker_color="teal",
+    ))
+    fig_hier.update_layout(
+        title=f"Topic Hierarchy ({run_key})",
+        height=700,
+        xaxis_title="Sentence Count",
+    )
+    with open(_ckpt(f"{run_key}_chart_hierarchy.html"), "w") as f:
+        f.write(fig_hier.to_html(include_plotlyjs="cdn", full_html=True))
+
+    # Chart 4: Heatmap of top-10 topic co-occurrence (cosine sim of centroids)
+    top10 = cluster_data[:10]
+    top10_centroids = np.array(list(map(lambda c: c["centroid"], top10)))
+    sim_matrix = cosine_similarity(top10_centroids)
+    top10_labels = list(map(lambda c: c["label"], top10))
+    fig_heat = px.imshow(
+        sim_matrix,
+        x=top10_labels, y=top10_labels,
+        color_continuous_scale="RdBu_r",
+        title=f"Topic Similarity Heatmap – Top 10 ({run_key})",
+    )
+    fig_heat.update_layout(height=500)
+    with open(_ckpt(f"{run_key}_chart_heatmap.html"), "w") as f:
+        f.write(fig_heat.to_html(include_plotlyjs="cdn", full_html=True))
+
+
+# ── Tool 3: label_topics_with_llm ─────────────────────────────────────────────
+@tool(handle_tool_error=True)
+def label_topics_with_llm(run_key: str) -> str:
+    """Send top MAX_LABEL_TOPICS topics to Mistral for labelling. Each topic gets:
+    label, category, confidence, reasoning, niche (true/false).
+    Saves labels.json. run_key: 'abstract' or 'title'."""
+    with open(_ckpt(f"{run_key}_summaries.json")) as f:
+        summaries = json.load(f)
+
+    top_topics = summaries[:MAX_LABEL_TOPICS]
+
+    topic_texts = "\n\n".join(list(map(
+        lambda t: (
+            f"Topic {t['topic_id']} ({t['sentence_count']} sentences):\n"
+            + "\n".join(list(map(lambda s: f"  - {s}", t["top_sentences"][:3])))
+        ),
+        top_topics,
+    )))
+
+    prompt = PromptTemplate.from_template(
+        """You are a research labelling expert. For each topic below, provide a JSON array.
+Each element must have: topic_id (int), label (research area name, max 6 words),
+category (broad domain), confidence (0.0-1.0), reasoning (1 sentence), niche (true/false).
+
+Return ONLY a valid JSON array. No markdown, no explanation.
+
+Topics:
+{topics}
+
+JSON array:"""
+    )
+
+    parser = JsonOutputParser()
+    chain = prompt | _llm() | parser
+    labeled = chain.invoke({"topics": topic_texts})
+
+    labeled_map = {item["topic_id"]: item for item in labeled}
+    result = list(map(
+        lambda t: {**t, **labeled_map.get(t["topic_id"], {})},
+        summaries,
+    ))
+
+    with open(_ckpt(f"{run_key}_labels.json"), "w") as f:
+        json.dump(result, f, indent=2)
+
+    labeled_count = len(labeled)
+    return (
+        f"✅ Labelling complete for **{run_key}** run.\n"
+        f"🏷️ Topics labeled: {labeled_count}\n"
+        f"📁 Saved: {run_key}_labels.json\n\n"
+        f"The review table has been populated with {labeled_count} labeled topics.\n"
+        f"**Please review the table below:** Edit the **Approve**, **Rename To**, and **Reasoning** columns, "
+        f"then click **Submit Review** to proceed to Phase 3."
+    )
+
+
+# ── Tool 4: consolidate_into_themes ───────────────────────────────────────────
+@tool(handle_tool_error=True)
+def consolidate_into_themes(run_key: str, theme_map: str) -> str:
+    """Merge researcher-approved topic groups into consolidated themes.
+    Recomputes centroids, recounts sentences and papers.
+    Saves themes.json.
+    run_key: 'abstract' or 'title'.
+    theme_map: JSON string mapping theme names to lists of topic_ids,
+               e.g. '{"AI Tourism": [0,1,5], "Smart Hotels": [2,3]}'"""
+    with open(_ckpt(f"{run_key}_labels.json")) as f:
+        labels = json.load(f)
+
+    theme_mapping = json.loads(theme_map)
+    label_lookup = {item["topic_id"]: item for item in labels}
+
+    themes = list(map(
+        lambda kv: _build_theme(kv[0], kv[1], label_lookup),
+        theme_mapping.items(),
+    ))
+    themes.sort(key=lambda t: t["sentence_count"], reverse=True)
+
+    with open(_ckpt(f"{run_key}_themes.json"), "w") as f:
+        json.dump(themes, f, indent=2)
+
+    return (
+        f"✅ Themes consolidated for **{run_key}** run.\n"
+        f"🗂️ Themes created: {len(themes)}\n"
+        + "\n".join(list(map(
+            lambda t: f"  • **{t['name']}**: {t['sentence_count']} sentences, {len(t['topic_ids'])} topics",
+            themes,
+        )))
+        + f"\n\n📁 Saved: {run_key}_themes.json\n\n"
+        f"**Please review the consolidated themes in the table.** "
+        f"Rename or adjust if needed, then click **Submit Review** to proceed to Phase 4."
+    )
+
+
+def _build_theme(name, topic_ids, label_lookup):
+    topics = list(filter(lambda t: t["topic_id"] in topic_ids, label_lookup.values()))
+    all_sents = sum(list(map(lambda t: t.get("top_sentences", []), topics)), [])
+    all_centroids = list(map(lambda t: t.get("centroid", []), topics))
+    centroid = np.mean(all_centroids, axis=0).tolist() if all_centroids else []
+    return {
+        "name": name,
+        "topic_ids": topic_ids,
+        "sentence_count": sum(list(map(lambda t: t.get("sentence_count", 0), topics))),
+        "top_sentences": all_sents[:NEAREST_K],
+        "centroid": centroid,
+        "pajais_match": "",
+        "match_confidence": 0.0,
+        "reasoning": "",
+        "is_novel": False,
+    }
+
+
+# ── Tool 5: compare_with_taxonomy ─────────────────────────────────────────────
+@tool(handle_tool_error=True)
+def compare_with_taxonomy(run_key: str) -> str:
+    """Map final themes to PAJAIS 25-category taxonomy using Mistral.
+    Each theme gets: pajais_match (or NOVEL), match_confidence, reasoning, is_novel.
+    Saves taxonomy_map.json. run_key: 'abstract' or 'title'."""
+    with open(_ckpt(f"{run_key}_themes.json")) as f:
+        themes = json.load(f)
+
+    theme_text = "\n".join(list(map(
+        lambda t: (
+            f"Theme: {t['name']}\n"
+            f"Evidence: {' | '.join(t.get('top_sentences', [])[:2])}"
+        ),
+        themes,
+    )))
+
+    pajais_text = "\n".join(PAJAIS_CATEGORIES)
+
+    prompt = PromptTemplate.from_template(
+        """You are a PAJAIS taxonomy expert. Map each research theme to the closest PAJAIS category.
+If no category fits well (similarity < 0.6), mark as NOVEL.
+
+PAJAIS Categories:
+{pajais}
+
+Themes to map:
+{themes}
+
+Return ONLY a JSON array. Each element: theme_name (str), pajais_match (str, exact category name or "NOVEL"),
+match_confidence (float 0-1), reasoning (str, 1 sentence), is_novel (bool).
+
+JSON array:"""
+    )
+
+    parser = JsonOutputParser()
+    chain = prompt | _llm() | parser
+    mapped = chain.invoke({"pajais": pajais_text, "themes": theme_text})
+
+    mapped_lookup = {item["theme_name"]: item for item in mapped}
+    result = list(map(
+        lambda t: {**t, **mapped_lookup.get(t["name"], {})},
+        themes,
+    ))
+
+    with open(_ckpt(f"{run_key}_taxonomy_map.json"), "w") as f:
+        json.dump(result, f, indent=2)
+
+    novel_count = len(list(filter(lambda t: t.get("is_novel", False), result)))
+    mapped_count = len(result) - novel_count
+
+    return (
+        f"✅ PAJAIS taxonomy mapping complete for **{run_key}** run.\n"
+        f"✅ MAPPED themes: {mapped_count}\n"
+        f"🆕 NOVEL themes: {novel_count}\n\n"
+        f"The review table now shows PAJAIS matches in the **Top Evidence** column.\n"
+        f"**Review the mapping in the table.** Novel themes may represent publishable research gaps. "
+        f"Click **Submit Review** to proceed to Phase 6."
+    )
+
+
+# ── Tool 6: generate_comparison_csv ───────────────────────────────────────────
+@tool(handle_tool_error=True)
+def generate_comparison_csv() -> str:
+    """Load themes from both abstract and title runs, create side-by-side comparison DataFrame.
+    Saves comparison.csv showing convergence and divergence between runs."""
+    with open(_ckpt("abstract_taxonomy_map.json")) as f:
+        abstract_themes = json.load(f)
+    with open(_ckpt("title_taxonomy_map.json")) as f:
+        title_themes = json.load(f)
+
+    abstract_rows = list(map(
+        lambda t: {
+            "Run": "Abstract",
+            "Theme": t["name"],
+            "Sentences": t.get("sentence_count", 0),
+            "PAJAIS Match": t.get("pajais_match", ""),
+            "Confidence": t.get("match_confidence", 0),
+            "Novel": t.get("is_novel", False),
+            "Reasoning": t.get("reasoning", ""),
+        },
+        abstract_themes,
+    ))
+    title_rows = list(map(
+        lambda t: {
+            "Run": "Title",
+            "Theme": t["name"],
+            "Sentences": t.get("sentence_count", 0),
+            "PAJAIS Match": t.get("pajais_match", ""),
+            "Confidence": t.get("match_confidence", 0),
+            "Novel": t.get("is_novel", False),
+            "Reasoning": t.get("reasoning", ""),
+        },
+        title_themes,
+    ))
+
+    df = pd.DataFrame(abstract_rows + title_rows)
+    df.to_csv(_ckpt("comparison.csv"), index=False)
+
+    return (
+        f"✅ Comparison CSV generated.\n"
+        f"📊 Abstract themes: {len(abstract_themes)}\n"
+        f"📊 Title themes: {len(title_themes)}\n"
+        f"📁 Saved: comparison.csv\n\n"
+        f"Check the **Download** tab for comparison.csv. "
+        f"Click **Submit Review** to confirm and generate the narrative report."
+    )
+
+
+# ── Tool 7: export_narrative ───────────────────────────────────────────────────
+@tool(handle_tool_error=True)
+def export_narrative(run_key: str) -> str:
+    """Generate a 500-word Section 7 narrative report for the literature review paper.
+    Uses themes and taxonomy mapping via Mistral. Saves narrative.txt.
+    run_key: 'abstract' or 'title'."""
+    with open(_ckpt(f"{run_key}_taxonomy_map.json")) as f:
+        themes = json.load(f)
+
+    themes_summary = "\n".join(list(map(
+        lambda t: (
+            f"- {t['name']}: {t.get('sentence_count', 0)} sentences, "
+            f"PAJAIS: {t.get('pajais_match', 'NOVEL')}, "
+            f"Novel: {t.get('is_novel', False)}"
+        ),
+        themes,
+    )))
+
+    prompt = PromptTemplate.from_template(
+        """You are an academic writing expert. Write a formal 500-word Section 7 (Thematic Analysis Results)
+for a journal literature review paper using the following data.
+
+Reference: Braun & Clarke (2006) six-phase thematic analysis methodology.
+Mention: BERTopic clustering, AgglomerativeClustering with cosine metric, Mistral LLM labelling.
+Include: key themes, PAJAIS taxonomy mapping, NOVEL themes as research gaps, limitations.
+Use academic language. Do not use bullet points — write in paragraphs.
+
+Themes and PAJAIS mapping ({run_key} run):
+{themes}
+
+Write Section 7 now (exactly 500 words):"""
+    )
+
+    chain = prompt | _llm()
+    narrative = chain.invoke({"run_key": run_key, "themes": themes_summary})
+
+    text = narrative.content if hasattr(narrative, "content") else str(narrative)
+
+    with open(_ckpt(f"{run_key}_narrative.txt"), "w") as f:
+        f.write(text)
+
+    return (
+        f"✅ Narrative report generated for **{run_key}** run.\n"
+        f"📝 500-word Section 7 draft saved.\n"
+        f"📁 Saved: {run_key}_narrative.txt\n\n"
+        f"Check the **Download** tab for all output files.\n\n"
+        f"**Phase 6 complete. Thematic analysis finished.**\n"
+        f"Download: comparison.csv, taxonomy_map.json, narrative.txt for your conference paper."
+    )
+
+
+# ── Exported tool list ─────────────────────────────────────────────────────────
+ALL_TOOLS = [
+    load_scopus_csv,
+    run_bertopic_discovery,
+    label_topics_with_llm,
+    consolidate_into_themes,
+    compare_with_taxonomy,
+    generate_comparison_csv,
+    export_narrative,
+]