Upload 4 files

agent.py

@@ -0,0 +1,143 @@
+"""
+agent.py — Braun & Clarke (2006) Thematic Analysis Agent.
+
+10 tools. 6 STOP gates. Reviewer approval after every interpretive output.
+Every number comes from a tool — the LLM never computes values.
+"""
+
+from langchain_mistralai import ChatMistralAI
+from langchain.agents import create_agent
+from langgraph.checkpoint.memory import InMemorySaver
+from tools import ALL_TOOLS
+
+SYSTEM_PROMPT = """
+You are a Braun & Clarke (2006) Computational Thematic Analysis Agent.
+
+RULES:
+  1. ONE PHASE PER MESSAGE.
+  2. ALL APPROVALS VIA REVIEW TABLE — never via chat. When review needed:
+       [WAITING FOR REVIEW TABLE]
+       Edit Approve / Rename To / Move To / Reasoning, then Submit Review.
+  3. NEVER FABRICATE DATA — every number, percentage, score, sentence list
+     MUST come from a tool. You CANNOT do arithmetic. If you need a number,
+     call a tool. If no tool exists for what you need, say so.
+  4. STOP GATES ARE ABSOLUTE — [FAILED] halts unconditionally.
+  5. EMIT PHASE STATUS at top of every response:
+       "[Phase X/6 | STOP Gates Passed: N/6 | Pending Review: Yes/No]"
+  6. TOOL ERRORS: log verbatim, identify cause, propose fix, wait.
+  7. AUTHOR KEYWORDS EXCLUDED from all embedding and clustering.
+  8. CHAT IS CONVERSATION, NOT DATA DUMP.
+     Your response in the chat window must be SHORT and CONVERSATIONAL:
+       - 3-5 sentences maximum summarising what you did
+       - State key numbers: "Found 45 clusters, 12 orphans"
+       - ALWAYS end with: "Results are loaded in the Review Table below."
+       - NEVER put markdown tables, JSON, raw data, or long lists in chat
+       - NEVER repeat the full tool output in chat
+     The Review Table (Section 3) auto-populates from your tool's
+     checkpoint files. The user sees the data THERE, not in chat.
+     Example good response:
+       "[Phase 2/6 | STOP Gates Passed: 0/6 | Pending Review: Yes]
+        I ran BERTopic discovery on 1,390 abstracts. Found 98 clusters
+        (min 3 members each) and 47 orphan sentences. Labelled the top
+        100 clusters via Mistral. Results are loaded in the Review Table
+        below. Please review and Submit when ready."
+     Example BAD response:
+       "[Phase 2/6 ...] Here are all 98 clusters: | # | Label | ...
+        (50 rows of markdown table dumped into chat)"
+
+10 TOOLS:
+  DETERMINISTIC (same input → same output):
+    1.  load_scopus_csv          — Phase 1: clean CSV, count, save .parquet
+    2.  run_bertopic_discovery   — Phase 2: embed + cluster (min 3 members)
+                                    + orphan report + 4 charts
+    4.  reassign_sentences       — Phase 2: move orphans/sentences between clusters
+    5.  consolidate_into_themes  — Phase 3: merge groups, recompute centroids
+    6.  compute_saturation       — Phase 4: coverage %, coherence, balance
+    7.  generate_theme_profiles  — Phase 5: top 5 nearest sentences per theme
+    9.  generate_comparison_csv  — Phase 6: abstract vs title joined on PAJAIS
+
+  LLM-DEPENDENT (grounded in real data, reviewer must approve):
+    3.  label_topics_with_llm    — Phase 2: Mistral names clusters
+    8.  compare_with_taxonomy    — Phase 5.5: map themes to PAJAIS 25
+    10. export_narrative         — Phase 6: 500-word Section 7
+
+B&C 6-PHASE METHODOLOGY:
+
+  PHASE 1 — FAMILIARISATION
+    The user message may contain a [CSV: /path/to/file.csv] prefix.
+    Extract the FULL path (everything between "CSV: " and "]") and pass
+    it as csv_path to load_scopus_csv. Do NOT modify or shorten the path.
+    Call load_scopus_csv. Show stats. STOP. Wait for "run abstract"/"run title".
+
+  PHASE 2 — INITIAL CODES
+    Call run_bertopic_discovery. Report: total clusters, orphan count.
+    If orphans > 0, tell reviewer: "N sentences did not fit any cluster
+    (minimum 3 members required). Review them and use Move To column."
+    Call label_topics_with_llm. Show top-20 labels.
+    STOP GATE 1: SG1-A (<5 topics), SG1-B (confidence <0.40),
+                 SG1-C (>40% generic), SG1-D (duplicates).
+    [WAITING FOR REVIEW TABLE]. STOP.
+    On Submit Review: if moves exist, call reassign_sentences.
+
+  PHASE 3 — THEMES
+    Parse review. Call consolidate_into_themes.
+    STOP GATE 2: SG2-A (<3 themes), SG2-B (singleton),
+                 SG2-C (duplicates), SG2-D (coverage <50%).
+    [WAITING FOR REVIEW TABLE]. STOP.
+
+  PHASE 4 — SATURATION
+    Call compute_saturation (NEVER compute these numbers yourself).
+    Present the EXACT numbers returned by the tool.
+    STOP GATE 3: SG3-A (coverage <60%), SG3-B (single theme >60%),
+                 SG3-C (coherence <0.30), SG3-D (<3 themes).
+    [WAITING FOR REVIEW TABLE]. STOP.
+
+  PHASE 5 — NAMING
+    Call generate_theme_profiles (NEVER recall sentences from memory).
+    Present the EXACT top-5 sentences returned by the tool per theme.
+    Propose names based on these real sentences.
+    [WAITING FOR REVIEW TABLE]. STOP.
+
+  PHASE 5.5 — PAJAIS MAPPING
+    Call compare_with_taxonomy.
+    STOP GATE 4: SG4-A (zero categories), SG4-B (>30% score <0.40),
+                 SG4-C (single category >50%), SG4-D (incomplete).
+    [WAITING FOR REVIEW TABLE]. STOP.
+
+  PHASE 6 — REPORT
+    Call generate_comparison_csv. Present convergence/divergence summary.
+    STOP GATE 5: Reviewer confirms comparison makes sense.
+    [WAITING FOR REVIEW TABLE]. STOP.
+    Call export_narrative. Present full 500-word draft.
+    STOP GATE 6: Reviewer approves final narrative.
+    [WAITING FOR REVIEW TABLE]. STOP.
+    DONE — all 6 gates passed.
+
+6 STOP GATES:
+  STOP-1 (Phase 2)   : Initial Code Quality
+  STOP-2 (Phase 3)   : Theme Coherence
+  STOP-3 (Phase 4)   : Saturation Adequacy
+  STOP-4 (Phase 5.5) : Taxonomy Alignment Quality
+  STOP-5 (Phase 6)   : Comparison Review         [NEW]
+  STOP-6 (Phase 6)   : Narrative Approval         [NEW]
+"""
+
+llm = ChatMistralAI(model="mistral-large-latest", temperature=0, max_tokens=8192)
+
+memory = InMemorySaver()
+
+agent = create_agent(
+    model=llm,
+    tools=ALL_TOOLS,
+    system_prompt=SYSTEM_PROMPT,
+    checkpointer=memory,
+)
+
+
+def run(user_message: str, thread_id: str = "default") -> str:
+    """Invoke the agent for one conversation turn."""
+    config  = {"configurable": {"thread_id": thread_id}}
+    payload = {"messages": [{"role": "user", "content": user_message}]}
+    result  = agent.invoke(payload, config=config)
+    msgs    = result.get("messages", [])
+    return (msgs and msgs[-1].content) or ""

app.py

@@ -0,0 +1,294 @@
+"""
+app.py — BERTopic Topic Modelling Agent UI.
+
+Three UX features:
+    1. Phase banner — large prominent display of current B&C phase
+    2. Dynamic prompts — phase-appropriate suggested next actions
+    3. Auto-populated review table — loads from tool checkpoint files
+
+9-column review table: #, Topic Label, Top Evidence, Sentences, Papers,
+Approve, Rename To, Move To, Reasoning.
+"""
+
+import gradio as gr
+import pandas as pd
+import json
+import os
+import re
+import tempfile
+from datetime import datetime
+from pathlib import Path
+from agent import run as agent_run
+
+THREAD_ID = f"bertopic-{datetime.now().strftime('%Y%m%d%H%M%S')}"
+
+REVIEW_COLS = [
+    "#", "Topic Label", "Top Evidence", "Sentences", "Papers",
+    "Approve", "Rename To", "Move To", "Reasoning",
+]
+
+EMPTY_TABLE = pd.DataFrame(columns=REVIEW_COLS)
+
+PHASE_INFO = {
+    0: ("Getting started", "⬜⬜⬜⬜⬜⬜",
+        "Upload a CSV file, then click **Analyze my Scopus CSV** and press Send"),
+    1: ("Phase 1 — Familiarisation", "🟦⬜⬜⬜⬜⬜",
+        "Click **Run abstract analysis** or **Run title analysis** and press Send"),
+    2: ("Phase 2 — Initial Codes", "🟦🟦⬜⬜⬜⬜",
+        "Review clusters in the **Results table** below. Edit Approve / Rename / Move, "
+        "then click **Submit Review**"),
+    3: ("Phase 3 — Themes", "🟦🟦🟦⬜⬜⬜",
+        "Review merged themes. Edit the table, then click **Submit Review**"),
+    4: ("Phase 4 — Saturation", "🟦🟦🟦🟦⬜⬜",
+        "Review saturation metrics. Click **Submit Review** to confirm"),
+    5: ("Phase 5 — Naming", "🟦🟦🟦🟦🟦⬜",
+        "Review theme profiles. Edit names, then **Submit Review**"),
+    6: ("Phase 6 — Report", "🟦🟦🟦🟦🟦🟦",
+        "Review comparison and narrative. **Submit Review** to finalise"),
+}
+
+_path = lambda file: str(
+    (hasattr(file, "name") and file.name)
+    or (isinstance(file, str) and file)
+    or ""
+)
+_name = lambda file: os.path.basename(_path(file))
+
+
+def _extract_phase(text: str) -> int:
+    """Extract phase number from agent response. Returns 0 if not found."""
+    found = re.findall(r"Phase (\d)", str(text))
+    return int((found or ["0"])[0])
+
+
+def _phase_banner(num: int) -> str:
+    """Generate prominent phase banner with progress bar and next step."""
+    name, progress, instruction = PHASE_INFO.get(num, PHASE_INFO[0])
+    return (
+        f"## {progress}  {name}\n\n"
+        f"**NEXT STEP →** {instruction}"
+    )
+
+
+def _load_review_table(base_dir: str) -> pd.DataFrame:
+    """Load latest checkpoint file into the 9-column review table.
+
+    Scans base_dir for topic_labels.json, themes.json, taxonomy_alignment.json,
+    summaries.json. Loads the most recently modified one and formats it.
+    Returns EMPTY_TABLE if nothing found.
+    """
+    base = Path(str(base_dir or ""))
+    candidates = sorted(
+        (
+            list(base.glob("topic_labels.json"))
+            + list(base.glob("themes.json"))
+            + list(base.glob("taxonomy_alignment.json"))
+            + list(base.glob("summaries.json"))
+        ),
+        key=lambda p: p.stat().st_mtime,
+        reverse=True,
+    ) * base.exists() or []
+
+    latest = (candidates[:1] or [None])[0]
+    return [EMPTY_TABLE.copy(), _format_checkpoint(latest)][bool(latest)]
+
+
+def _format_checkpoint(path) -> pd.DataFrame:
+    """Format a checkpoint JSON file into review table rows."""
+    raw = json.loads(Path(path).read_text())
+    data = (isinstance(raw, dict) and raw.get("clusters", raw.get("per_theme", []))) or \
+           (isinstance(raw, list) and raw) or []
+
+    def _row(item: dict) -> dict:
+        """Map one JSON item to review table columns."""
+        return {
+            "#":            item.get("topic_id", item.get("theme_id", 0)),
+            "Topic Label":  item.get("label", item.get("theme_label", "")),
+            "Top Evidence":  str(item.get("representative",
+                                  item.get("notes", "")))[:150],
+            "Sentences":    item.get("size", item.get("total_papers",
+                                    item.get("papers", 0))),
+            "Papers":       item.get("size", item.get("total_papers",
+                                    item.get("papers", 0))),
+            "Approve":      "",
+            "Rename To":    "",
+            "Move To":      "",
+            "Reasoning":    str(item.get("rationale", item.get("notes", ""))),
+        }
+
+    rows = list(map(_row, data[:50]))
+    return [EMPTY_TABLE.copy(), pd.DataFrame(rows, columns=REVIEW_COLS)][bool(rows)]
+
+
+def on_file_upload(file):
+    """Extract CSV stats and store base directory."""
+    path = _path(file)
+    result = (not path) and ("Upload a CSV to begin.", "", _phase_banner(0))
+    return result or _do_file_upload(path, file)
+
+
+def _do_file_upload(path: str, file) -> tuple:
+    """Actual file processing after path validation."""
+    df = pd.read_csv(path)
+    rows, cols = df.shape
+    base = str(Path(path).parent)
+    info = (
+        f"**Loaded:** `{_name(file)}`\n\n"
+        f"**Shape:** {rows:,} rows x {cols} columns\n\n"
+        f"**Columns:** {', '.join(df.columns[:6].tolist())}\n\n"
+        f"*Click a prompt below and press Send to begin.*"
+    )
+    return info, base, _phase_banner(1)
+
+
+def on_send(user_msg, history, file, base_dir):
+    """Pass user message to agent. Update phase banner and review table."""
+    msg = (user_msg or "").strip() or "help"
+    csv_tag = f"[CSV: {_path(file)}]\n" * bool(file)
+
+    history = list(history or [])
+    history.append({"role": "user", "content": msg})
+    history.append({"role": "assistant", "content": "Thinking..."})
+    yield history, "", gr.skip(), gr.skip(), gr.skip()
+
+    reply = agent_run(csv_tag + msg, thread_id=THREAD_ID)
+    history[-1] = {"role": "assistant", "content": reply}
+
+    phase = _extract_phase(reply)
+    banner = _phase_banner(phase)
+    table = _load_review_table(base_dir)
+
+    yield history, "", banner, table, base_dir
+
+
+def on_submit_review(table_df, history, base_dir):
+    """Serialise review table edits to agent."""
+    history = list(history or [])
+    edits = table_df.to_json(orient="records", indent=2)
+
+    history.append({"role": "user", "content": "[REVIEW SUBMITTED]"})
+    history.append({"role": "assistant", "content": "Processing review..."})
+
+    reply = agent_run(
+        "Reviewer submitted table edits.\n\n"
+        f"```json\n{edits}\n```\n\n"
+        "Process: Approve/Reject decisions, Rename To values, "
+        "Move To reassignments (call reassign_sentences if moves exist), "
+        "Reasoning notes. Then check STOP gates and proceed.",
+        thread_id=THREAD_ID,
+    )
+    history[-1] = {"role": "assistant", "content": reply}
+
+    phase = _extract_phase(reply)
+    return history, _phase_banner(phase), _load_review_table(base_dir)
+
+
+def on_download(table_df, history):
+    """Export review CSV and chat TXT."""
+    csv_tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".csv", prefix="review_")
+    table_df.to_csv(csv_tmp.name, index=False)
+
+    txt_tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".txt", prefix="chat_")
+    txt_tmp.write(
+        "\n\n".join(
+            list(map(
+                lambda m: f"{m.get('role', '').upper()}: {m.get('content', '')}",
+                history or [],
+            ))
+        ).encode("utf-8")
+    )
+    txt_tmp.close()
+    return [csv_tmp.name, txt_tmp.name]
+
+
+with gr.Blocks(title="BERTopic Agent") as demo:
+
+    base_dir_state = gr.State(value="")
+
+    gr.Markdown("# BERTopic Modelling Agent")
+    gr.Markdown(
+        "**Braun & Clarke 6-Phase Thematic Analysis** "
+        "| 10 Tools | 6 STOP Gates | Cosine Agglomerative Clustering"
+    )
+
+    phase_banner = gr.Markdown(value=_phase_banner(0))
+
+    gr.Markdown("---\n### Section 1 — Data input")
+    with gr.Row():
+        with gr.Column(scale=3):
+            file_input = gr.File(
+                label="Upload Scopus CSV",
+                file_types=[".csv"],
+                file_count="single",
+            )
+        with gr.Column(scale=5):
+            file_info = gr.Markdown("Upload a CSV to begin.")
+
+    gr.Markdown("---\n### Section 2 — Agent conversation")
+    chatbot = gr.Chatbot(label="BERTopic Agent", height=200)
+    with gr.Row():
+        msg_box = gr.Textbox(
+            placeholder="Type a message or click a prompt below, then press Send",
+            show_label=False, scale=7, lines=1,
+        )
+        send_btn = gr.Button("Send", variant="primary", scale=1)
+
+    gr.Examples(
+        examples=[
+            "Analyze my Scopus CSV",
+            "Run abstract analysis",
+            "Run title analysis",
+            "Proceed to next phase",
+            "Show corpus statistics",
+        ],
+        inputs=msg_box,
+        label="Quick prompts (click to fill, then press Send)",
+    )
+
+    gr.Markdown("---\n### Section 3 — Results (auto-populated from tool outputs)")
+    gr.Markdown(
+        "This table fills automatically when the agent runs tools. "
+        "Edit **Approve**, **Rename To**, **Move To**, **Reasoning** columns, "
+        "then click **Submit Review**."
+    )
+    review_table = gr.Dataframe(
+        value=EMPTY_TABLE,
+        headers=REVIEW_COLS,
+        datatype=["number", "str", "str", "number", "number",
+                  "str", "str", "str", "str"],
+        column_count=(9, "fixed"),
+        interactive=True, wrap=True,
+    )
+    with gr.Row():
+        clear_btn = gr.Button("Clear table", variant="secondary", scale=2)
+        sub_btn = gr.Button("Submit Review", variant="primary", scale=4)
+
+    with gr.Accordion("Download", open=False):
+        dl_btn = gr.Button("Generate downloads", variant="primary")
+        dl_files = gr.File(label="Downloads", file_count="multiple",
+                          interactive=False)
+
+    file_input.change(
+        on_file_upload,
+        inputs=[file_input],
+        outputs=[file_info, base_dir_state, phase_banner],
+    )
+    send_btn.click(
+        on_send,
+        inputs=[msg_box, chatbot, file_input, base_dir_state],
+        outputs=[chatbot, msg_box, phase_banner, review_table, base_dir_state],
+    )
+    msg_box.submit(
+        on_send,
+        inputs=[msg_box, chatbot, file_input, base_dir_state],
+        outputs=[chatbot, msg_box, phase_banner, review_table, base_dir_state],
+    )
+    clear_btn.click(lambda: EMPTY_TABLE.copy(), outputs=[review_table])
+    sub_btn.click(
+        on_submit_review,
+        inputs=[review_table, chatbot, base_dir_state],
+        outputs=[chatbot, phase_banner, review_table],
+    )
+    dl_btn.click(on_download, inputs=[review_table, chatbot], outputs=[dl_files])
+
+demo.launch(ssr_mode=False, theme=gr.themes.Soft())

requirements.txt

@@ -0,0 +1,10 @@
+gradio>=6.0.0
+langchain>=1.0.0
+langchain-mistralai>=1.0.0
+langgraph>=1.0.0
+sentence-transformers>=3.0.0
+scikit-learn>=1.4.0
+numpy>=1.26.0
+pandas>=2.1.0
+plotly>=5.20.0
+pyarrow>=15.0.0

tools.py

@@ -0,0 +1,999 @@
+"""
+tools.py — 10 @tool functions for BERTopic-style topic modelling.
+
+Pipeline (called in this order by the LLM agent):
+
+    1.  load_scopus_csv          — ingest CSV, strip boilerplate, save .parquet
+    2.  run_bertopic_discovery   — embed → cosine agglomerative cluster (min 3
+                                    members) → centroids → orphan report → 4 charts
+    3.  label_topics_with_llm    — Mistral labels top 100 clusters
+    4.  reassign_sentences       — move orphan/misplaced sentences between clusters
+    5.  consolidate_into_themes  — merge reviewer-approved groups
+    6.  compute_saturation       — coverage %, coherence, balance per theme
+    7.  generate_theme_profiles  — top 5 nearest sentences per theme centroid
+    8.  compare_with_taxonomy    — map themes to PAJAIS 25 categories
+    9.  generate_comparison_csv  — abstract vs title side-by-side
+    10. export_narrative         — 500-word Section 7 via Mistral
+
+Design rules:
+
+    Every number, percentage, score, or list of sentences presented to the
+    reviewer MUST come from a tool — never from the LLM's imagination.
+
+    Deterministic tools (1,2,4,5,6,7,9): same input → same output, every run.
+    LLM-dependent tools (3,8,10): grounded in real data passed via prompt,
+    but labels/mappings/narrative may vary slightly between runs.
+    All LLM-dependent outputs require reviewer approval before advancing.
+
+    ZERO if/elif/else — all decisions by the LLM
+    ZERO for/while    — list(map(...)) and numpy vectorised ops
+    ZERO try/except   — errors surface to the LLM via ToolNode
+
+Constants reference:
+
+    EMBED_MODEL = "all-MiniLM-L6-v2"
+        384d sentence embeddings. Runs locally, no API calls.
+        normalize_embeddings=True → cosine similarity = dot product.
+
+    CLUSTER_THRESHOLD = 0.7
+        Agglomerative stops merging at cosine distance > 0.7.
+        0.5 → ~2,102 clusters. 0.7 → ~100 clusters. 0.8 → ~30 clusters.
+
+    MIN_CLUSTER_SIZE = 3
+        Clusters with fewer than 3 members are dissolved. Their sentences
+        become orphans (label=-1) reported to the reviewer for reassignment.
+
+    N_CENTROIDS = 5
+        Top clusters extracted for initial discovery report and charts.
+
+    TOP_TOPICS_LLM = 100
+        Maximum clusters sent to Mistral for labelling.
+
+    NARRATIVE_WORDS = 500
+        Target word count for Section 7 narrative.
+
+    PAJAIS_25
+        25 IS research categories from Jiang et al. (2019).
+        Used in Phase 5.5 for taxonomy alignment.
+
+    BOILERPLATE_PATTERNS (9 regexes)
+        Strip publisher noise: copyright, DOI, Elsevier, Springer,
+        IEEE, Wiley, Taylor & Francis.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+
+from pathlib import Path
+from langchain_core.tools import tool
+from langchain_mistralai import ChatMistralAI
+from langchain_core.prompts import PromptTemplate
+from langchain_core.output_parsers import JsonOutputParser
+from sentence_transformers import SentenceTransformer
+from sklearn.cluster import AgglomerativeClustering
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.preprocessing import normalize
+from sklearn.decomposition import PCA
+
+
+RUN_CONFIGS = {
+    "abstract": ["Abstract"],
+    "title":    ["Title"],
+}
+
+PAJAIS_25 = [
+    "Accounting Information Systems",
+    "Artificial Intelligence & Expert Systems",
+    "Big Data & Analytics",
+    "Business Intelligence & Decision Support",
+    "Cloud Computing",
+    "Cybersecurity & Privacy",
+    "Database Management",
+    "Digital Transformation",
+    "E-Business & E-Commerce",
+    "Enterprise Resource Planning",
+    "Fintech & Digital Finance",
+    "Geographic Information Systems",
+    "Health Informatics",
+    "Human-Computer Interaction",
+    "Information Systems Development",
+    "IT Governance & Management",
+    "IT Strategy & Competitive Advantage",
+    "Knowledge Management",
+    "Machine Learning & Deep Learning",
+    "Mobile Computing",
+    "Natural Language Processing",
+    "Recommender Systems",
+    "Social Media & Web 2.0",
+    "Supply Chain & Logistics IS",
+    "Virtual Reality & Augmented Reality",
+]
+
+BOILERPLATE_PATTERNS = [
+    r"©\s*\d{4}",
+    r"all rights reserved",
+    r"published by elsevier",
+    r"this article is protected",
+    r"doi:\s*10\.\d{4,}",
+    r"springer nature",
+    r"ieee xplore",
+    r"wiley online library",
+    r"taylor & francis",
+]
+
+BOILERPLATE_RE    = re.compile("|".join(BOILERPLATE_PATTERNS), flags=re.IGNORECASE)
+SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+")
+EMBED_MODEL       = "all-MiniLM-L6-v2"
+N_CENTROIDS       = 5
+CLUSTER_THRESHOLD = 0.7
+MIN_CLUSTER_SIZE  = 3
+TOP_TOPICS_LLM    = 100
+NARRATIVE_WORDS   = 500
+
+
+def _clean_text(text: str) -> str:
+    """Remove publisher boilerplate from a single text string.
+
+    Applies 9-pattern BOILERPLATE_RE regex to strip copyright notices,
+    DOI prefixes, and publisher tags that would pollute embeddings.
+
+    Args:
+        text: Raw abstract or title string.
+
+    Returns:
+        Cleaned string with boilerplate removed and whitespace trimmed.
+    """
+    return BOILERPLATE_RE.sub("", str(text)).strip()
+
+
+def _sentence_count(text: str) -> int:
+    """Count sentences using regex split on terminal punctuation.
+
+    Args:
+        text: Cleaned abstract or title text.
+
+    Returns:
+        Number of sentences (minimum 1 for any non-empty input).
+    """
+    return len(SENTENCE_SPLIT_RE.split(text.strip()))
+
+
+def _embed(texts: list[str]) -> np.ndarray:
+    """Embed texts into 384d L2-normalized unit vectors.
+
+    Uses SentenceTransformer('all-MiniLM-L6-v2') locally — no API calls.
+    normalize_embeddings=True ensures cosine_similarity = dot product.
+
+    Args:
+        texts: List of N cleaned text strings.
+
+    Returns:
+        np.ndarray shape (N, 384), dtype float32, L2-normalized.
+    """
+    model = SentenceTransformer(EMBED_MODEL)
+    raw   = model.encode(texts, show_progress_bar=False, normalize_embeddings=True)
+    return np.array(raw, dtype=np.float32)
+
+
+def _cosine_cluster(matrix: np.ndarray, threshold: float, min_size: int) -> np.ndarray:
+    """Cluster embeddings using agglomerative cosine clustering.
+
+    Works DIRECTLY in 384d space — no UMAP. After clustering, any cluster
+    with fewer than min_size members is dissolved: its sentences get
+    label=-1 (orphan) and are reported to the reviewer for reassignment.
+
+    Algorithm:
+        1. Start: every text is its own cluster.
+        2. Merge the two closest clusters (average cosine distance).
+        3. Repeat until smallest distance exceeds threshold.
+        4. Post-process: dissolve clusters smaller than min_size.
+
+    Args:
+        matrix:    (N, 384) embedding matrix, L2-normalized.
+        threshold: Max cosine distance for merging (0.7 → ~100 clusters).
+        min_size:  Minimum members per cluster (3). Smaller → orphan.
+
+    Returns:
+        np.ndarray shape (N,) with integer labels. -1 = orphan.
+    """
+    normed = normalize(matrix, norm="l2")
+    model  = AgglomerativeClustering(
+        n_clusters=None,
+        metric="cosine",
+        linkage="average",
+        distance_threshold=threshold,
+    )
+    labels = model.fit_predict(normed).astype(int)
+    unique, counts = np.unique(labels, return_counts=True)
+    small_clusters = unique[counts < min_size]
+    return np.where(np.isin(labels, small_clusters), -1, labels)
+
+
+def _centroid(vecs: np.ndarray) -> np.ndarray:
+    """Compute L2-normalized centroid (average direction in 384d space).
+
+    Args:
+        vecs: (M, 384) matrix of member embeddings for one cluster.
+
+    Returns:
+        1d np.ndarray shape (384,), L2-normalized.
+    """
+    return normalize(vecs.mean(axis=0, keepdims=True), norm="l2")[0]
+
+
+def _top_n_centroids(matrix: np.ndarray, labels: np.ndarray, n: int) -> list[dict]:
+    """Extract N largest clusters by size and compute their centroids.
+
+    Excludes orphans (label=-1) from the ranking.
+
+    Args:
+        matrix: (N, 384) full embedding matrix.
+        labels: (N,) integer cluster labels (-1 = orphan).
+        n:      How many top clusters to return.
+
+    Returns:
+        List of N dicts with: label, size, indices, centroid.
+    """
+    valid_mask = labels >= 0
+    valid_labels = labels[valid_mask]
+    unique, counts = np.unique(valid_labels, return_counts=True)
+    order      = np.argsort(counts)[::-1][:n]
+    top_labels = unique[order]
+
+    def _build(lbl: int) -> dict:
+        """Build summary dict for one cluster."""
+        idx = np.where(labels == lbl)[0].tolist()
+        return {
+            "label":    int(lbl),
+            "size":     len(idx),
+            "indices":  idx,
+            "centroid": _centroid(matrix[idx]),
+        }
+
+    return list(map(_build, top_labels))
+
+
+def _mistral_chain(template_str: str):
+    """Create PromptTemplate → ChatMistralAI → JsonOutputParser chain.
+
+    Args:
+        template_str: Prompt template with {variable} placeholders.
+
+    Returns:
+        LangChain Runnable chain that accepts dict and returns parsed JSON.
+    """
+    llm    = ChatMistralAI(model="mistral-large-latest", temperature=0)
+    prompt = PromptTemplate.from_template(template_str)
+    return prompt | llm | JsonOutputParser()
+
+
+def _dark_layout(title: str) -> dict:
+    """Return Plotly layout dict with dark theme styling.
+
+    Args:
+        title: Chart title string.
+
+    Returns:
+        Dict for fig.update_layout(**_dark_layout("...")).
+    """
+    return dict(
+        title=title, paper_bgcolor="#0F172A", plot_bgcolor="#0F172A",
+        font=dict(color="#CBD5E1", family="Sora,sans-serif"),
+        margin=dict(t=50, b=40, l=40, r=20),
+    )
+
+
+@tool
+def load_scopus_csv(csv_path: str, run_mode: str = "abstract") -> str:
+    """Load a Scopus CSV, count papers/sentences, apply boilerplate filter.
+
+    Phase 1 — Familiarisation. DETERMINISTIC.
+
+    Steps:
+        1. Read CSV, drop rows where target column is null
+        2. Apply 9-pattern boilerplate regex to clean each text
+        3. Count sentences per paper
+        4. Save cleaned DataFrame as .parquet
+
+    Args:
+        csv_path: Path to raw Scopus CSV.
+        run_mode: 'abstract' or 'title'.
+
+    Returns:
+        JSON: total_papers, total_sentences, columns_used,
+        boilerplate_removed, cleaned_parquet, run_mode.
+    """
+    cols   = RUN_CONFIGS[run_mode]
+    target = cols[0]
+
+    df            = pd.read_csv(csv_path).dropna(subset=[target]).reset_index(drop=True)
+    raw_texts     = df[target].tolist()
+    cleaned_texts = list(map(_clean_text, raw_texts))
+
+    boilerplate_removed = sum(map(
+        lambda pair: int(pair[0] != pair[1]),
+        zip(raw_texts, cleaned_texts),
+    ))
+
+    df[f"{target}_clean"] = cleaned_texts
+    df["sentence_count"]  = list(map(_sentence_count, cleaned_texts))
+
+    out_path = Path(csv_path).with_suffix(".clean.parquet")
+    df.to_parquet(out_path, index=False)
+
+    return json.dumps({
+        "total_papers":        len(df),
+        "total_sentences":     int(df["sentence_count"].sum()),
+        "columns_used":        cols,
+        "boilerplate_removed": boilerplate_removed,
+        "cleaned_parquet":     str(out_path),
+        "run_mode":            run_mode,
+    }, indent=2)
+
+
+@tool
+def run_bertopic_discovery(parquet_path: str, run_mode: str = "abstract") -> str:
+    """Embed texts, cluster them, report orphans, generate charts.
+
+    Phase 2 — Initial Codes. DETERMINISTIC.
+
+    Steps:
+        1. Load cleaned parquet, drop Author Keywords columns (RULE 8)
+        2. Embed all texts → N x 384 matrix of unit vectors
+        3. Save embedding matrix as .emb.npy
+        4. Cluster in 384d space (NO UMAP), min 3 members per cluster
+        5. Sentences in clusters < 3 members become orphans (label=-1)
+        6. Extract top-5 clusters, compute centroids
+        7. Save summaries.json with clusters + orphan list
+        8. Generate 4 Plotly HTML charts
+
+    Args:
+        parquet_path: Path to .clean.parquet from load_scopus_csv.
+        run_mode:     'abstract' or 'title'.
+
+    Returns:
+        JSON: total_clusters, orphan_count, summaries_json, embeddings_npy,
+        charts dict.
+    """
+    cols   = RUN_CONFIGS[run_mode]
+    target = f"{cols[0]}_clean"
+
+    df = pd.read_parquet(parquet_path).drop(
+        columns=[c for c in pd.read_parquet(parquet_path).columns
+                 if re.search(r"keyword|author", c, re.I)],
+        errors="ignore",
+    )
+
+    texts      = df[target].tolist()
+    embeddings = _embed(texts)
+    base       = Path(parquet_path).parent
+
+    np.save(str(base / Path(parquet_path).stem) + ".emb.npy", embeddings)
+
+    labels        = _cosine_cluster(embeddings, CLUSTER_THRESHOLD, MIN_CLUSTER_SIZE)
+    orphan_idx    = np.where(labels == -1)[0].tolist()
+    orphan_count  = len(orphan_idx)
+    valid_count   = int((labels >= 0).sum())
+    n_clusters    = int(np.unique(labels[labels >= 0]).shape[0])
+    top_centroids = _top_n_centroids(embeddings, labels, N_CENTROIDS)
+
+    def _topic_row(tc: dict) -> dict:
+        """Convert centroid dict into summary row for summaries.json."""
+        return {
+            "topic_id":       tc["label"],
+            "size":           tc["size"],
+            "representative": texts[tc["indices"][0]][:200],
+            "indices":        tc["indices"],
+        }
+
+    summaries = list(map(_topic_row, top_centroids))
+
+    orphans = list(map(
+        lambda i: {"sentence_idx": int(i), "text": texts[i][:200]},
+        orphan_idx,
+    ))
+
+    output = {"clusters": summaries, "orphans": orphans}
+    (base / "summaries.json").write_text(json.dumps(output, indent=2))
+
+    unique, counts = np.unique(labels[labels >= 0], return_counts=True)
+    order          = np.argsort(counts)[::-1][:20]
+    c1 = go.Figure(go.Bar(
+        x=list(map(str, unique[order])), y=counts[order].tolist(),
+        marker_color="#3B82F6", text=counts[order].tolist(), textposition="outside",
+    ))
+    c1.update_layout(**_dark_layout("Topic Size Distribution (Top 20)"),
+                     xaxis=dict(showgrid=False),
+                     yaxis=dict(showgrid=True, gridcolor="#1E293B"))
+    c1.write_html(str(base / "chart_topic_sizes.html"))
+
+    centroid_matrix = np.vstack([tc["centroid"] for tc in top_centroids])
+    sim_matrix      = cosine_similarity(centroid_matrix)
+    clabels         = list(map(lambda tc: f"T{tc['label']}", top_centroids))
+    c2 = go.Figure(go.Heatmap(z=sim_matrix, x=clabels, y=clabels, colorscale="Blues"))
+    c2.update_layout(**_dark_layout("Top-5 Centroid Cosine Similarity"))
+    c2.write_html(str(base / "chart_centroid_heatmap.html"))
+
+    sc = df.get("sentence_count", pd.Series([0] * len(df))).tolist()
+    c3 = go.Figure(go.Histogram(x=sc, nbinsx=40, marker_color="#22D3EE"))
+    c3.update_layout(**_dark_layout("Sentence Count Distribution"),
+                     xaxis=dict(showgrid=False),
+                     yaxis=dict(showgrid=True, gridcolor="#1E293B"))
+    c3.write_html(str(base / "chart_sentence_distribution.html"))
+
+    coords     = PCA(n_components=2).fit_transform(centroid_matrix)
+    point_text = list(map(lambda tc: f"T{tc['label']}({tc['size']})", top_centroids))
+    c4 = go.Figure(go.Scatter(
+        x=coords[:, 0].tolist(), y=coords[:, 1].tolist(),
+        mode="markers+text", text=point_text, textposition="top center",
+        marker=dict(size=12, color="#F59E0B", line=dict(width=1, color="#0F172A")),
+    ))
+    c4.update_layout(**_dark_layout("Top-5 Centroids — PCA Projection"))
+    c4.write_html(str(base / "chart_centroid_pca.html"))
+
+    emb_path = str(base / Path(parquet_path).stem) + ".emb.npy"
+    return json.dumps({
+        "total_clusters":  n_clusters,
+        "orphan_count":    orphan_count,
+        "valid_sentences": valid_count,
+        "top_centroids":   N_CENTROIDS,
+        "summaries_json":  str(base / "summaries.json"),
+        "embeddings_npy":  emb_path,
+        "needs_review":    True,
+        "charts": {
+            "topic_sizes":      str(base / "chart_topic_sizes.html"),
+            "centroid_heatmap": str(base / "chart_centroid_heatmap.html"),
+            "sentence_dist":    str(base / "chart_sentence_distribution.html"),
+            "centroid_pca":     str(base / "chart_centroid_pca.html"),
+        },
+    }, indent=2)
+
+
+@tool
+def label_topics_with_llm(summaries_json_path: str) -> str:
+    """Send top-100 topic summaries to Mistral for labelling.
+
+    Phase 2 — Labelling. LLM-DEPENDENT (grounded in real sentences).
+
+    Steps:
+        1. Load summaries.json clusters (not orphans)
+        2. Take top 100 by size
+        3. Mistral reads representative sentences → assigns labels
+        4. Returns: topic_id, label, rationale, confidence per cluster
+        5. Save as topic_labels.json
+
+    Args:
+        summaries_json_path: Path to summaries.json.
+
+    Returns:
+        JSON: labelled_topics count + output path. needs_review=True.
+    """
+    data          = json.loads(Path(summaries_json_path).read_text())
+    summaries     = data.get("clusters", data)[:TOP_TOPICS_LLM]
+
+    template = (
+        "You are a scientific topic labelling expert.\n\n"
+        "Below are {n} topic summaries from a BERTopic analysis of academic papers.\n"
+        "Each summary has: topic_id, size, representative text.\n\n"
+        "{summaries}\n\n"
+        "For EACH topic return a JSON array where every element has:\n"
+        "  topic_id   : integer (copy from input)\n"
+        "  label      : 2-5 word snake_case topic label\n"
+        "  rationale  : one sentence justification\n"
+        "  confidence : float 0.0-1.0\n\n"
+        "Return ONLY the JSON array — no markdown, no preamble."
+    )
+
+    result   = _mistral_chain(template).invoke({
+        "n":         len(summaries),
+        "summaries": json.dumps(summaries, indent=2),
+    })
+    out_path = Path(summaries_json_path).parent / "topic_labels.json"
+    out_path.write_text(json.dumps(result, indent=2))
+
+    return json.dumps({
+        "labelled_topics": len(result),
+        "output":          str(out_path),
+        "needs_review":    True,
+    }, indent=2)
+
+
+@tool
+def reassign_sentences(
+    summaries_json_path: str,
+    embeddings_npy_path: str,
+    move_instructions: str,
+) -> str:
+    """Move orphan or misplaced sentences between clusters.
+
+    Phase 2 — Orphan handling. DETERMINISTIC.
+
+    The reviewer specifies moves as JSON:
+        [{"sentence_idx": 42, "to_cluster": 3},
+         {"sentence_idx": 99, "to_cluster": "new"}]
+
+    For "new" targets, a fresh cluster ID is assigned.
+    After all moves, centroids are recomputed for affected clusters.
+
+    Steps:
+        1. Load summaries.json and embeddings
+        2. Parse move instructions
+        3. Update cluster assignments
+        4. Recompute centroids for affected clusters
+        5. Save updated summaries.json
+
+    Args:
+        summaries_json_path: Path to summaries.json.
+        embeddings_npy_path: Path to .emb.npy.
+        move_instructions:   JSON array of {sentence_idx, to_cluster} dicts.
+
+    Returns:
+        JSON: moves_applied count, orphans_remaining, updated summaries path.
+    """
+    data       = json.loads(Path(summaries_json_path).read_text())
+    embeddings = np.load(embeddings_npy_path)
+    moves      = json.loads(move_instructions)
+    clusters   = data.get("clusters", [])
+    orphans    = data.get("orphans", [])
+
+    all_indices = {}
+    list(map(
+        lambda c: all_indices.update({idx: c["topic_id"] for idx in c.get("indices", [])}),
+        clusters,
+    ))
+
+    max_id = max(map(lambda c: c.get("topic_id", 0), clusters), default=0)
+    new_id_counter = [max_id + 1]
+
+    def _apply_move(m: dict) -> dict:
+        """Apply one move instruction, return the resolved target cluster ID."""
+        s_idx = m["sentence_idx"]
+        target = m["to_cluster"]
+        resolved = (target == "new") and new_id_counter.__setitem__(0, new_id_counter[0] + 1) or target
+        final_id = new_id_counter[0] - 1 * (target == "new") + target * (target != "new")
+        all_indices[s_idx] = int(target) * (target != "new") + new_id_counter[0] * (target == "new")
+        return {"sentence_idx": s_idx, "assigned_to": all_indices[s_idx]}
+
+    applied = list(map(_apply_move, moves))
+
+    unique_clusters = set(all_indices.values())
+
+    def _rebuild_cluster(cid: int) -> dict:
+        """Rebuild a cluster dict from the updated index map."""
+        idx = [k for k, v in all_indices.items() if v == cid]
+        vecs = embeddings[idx or [0]]
+        return {
+            "topic_id":       int(cid),
+            "size":           len(idx),
+            "representative": "",
+            "indices":        idx,
+            "centroid":       _centroid(vecs).tolist(),
+        }
+
+    updated_clusters = list(map(_rebuild_cluster, sorted(unique_clusters)))
+    remaining_orphan_idx = [o["sentence_idx"] for o in orphans
+                           if o["sentence_idx"] not in all_indices]
+
+    output = {
+        "clusters": updated_clusters,
+        "orphans":  list(map(
+            lambda i: {"sentence_idx": i, "text": ""},
+            remaining_orphan_idx,
+        )),
+    }
+    Path(summaries_json_path).write_text(json.dumps(output, indent=2))
+
+    return json.dumps({
+        "moves_applied":     len(applied),
+        "orphans_remaining": len(remaining_orphan_idx),
+        "summaries_json":    summaries_json_path,
+        "needs_review":      True,
+    }, indent=2)
+
+
+@tool
+def consolidate_into_themes(
+    labels_json_path: str,
+    embeddings_npy_path: str,
+    approved_topic_ids: str,
+) -> str:
+    """Merge approved topic groups into consolidated themes.
+
+    Phase 3 — Theme Search. DETERMINISTIC.
+
+    Steps:
+        1. Load topic_labels.json and embedding matrix
+        2. Parse approved groupings (JSON array of arrays)
+        3. Pool all member embeddings per group
+        4. Compute fresh L2-normalized centroid per merged group
+        5. Build theme name from joined sub-labels
+        6. Save themes.json
+
+    Args:
+        labels_json_path:    Path to topic_labels.json.
+        embeddings_npy_path: Path to .emb.npy.
+        approved_topic_ids:  JSON array of arrays, e.g. [[0,1,2],[3,4],[5]].
+
+    Returns:
+        JSON: themes_created count + themes_json path. needs_review=True.
+    """
+    labels_data = json.loads(Path(labels_json_path).read_text())
+    embeddings  = np.load(embeddings_npy_path)
+    groups      = json.loads(approved_topic_ids)
+    label_map   = {item["topic_id"]: item for item in labels_data}
+
+    def _merge_group(group_ids: list[int]) -> dict:
+        """Merge topic IDs into one theme, recompute centroid."""
+        members    = [m for m in map(label_map.get, group_ids) if m is not None]
+        all_idx    = sum(map(lambda m: m.get("indices", []), members), [])
+        vecs       = embeddings[all_idx or [0]]
+        centroid   = _centroid(vecs)
+        sub_labels = list(map(lambda m: m.get("label", ""), members))
+        theme_name = "_".join(
+            dict.fromkeys(sum(map(lambda lbl: lbl.split("_"), sub_labels), []))
+        )[:60]
+        return {
+            "theme_id":     group_ids[0],
+            "theme_label":  theme_name,
+            "merged_ids":   group_ids,
+            "total_papers": len(set(all_idx)),
+            "indices":      all_idx,
+            "centroid":     centroid.tolist(),
+        }
+
+    themes   = list(map(_merge_group, groups))
+    out_path = Path(labels_json_path).parent / "themes.json"
+    out_path.write_text(json.dumps(themes, indent=2))
+
+    return json.dumps({
+        "themes_created": len(themes),
+        "themes_json":    str(out_path),
+        "needs_review":   True,
+    }, indent=2)
+
+
+@tool
+def compute_saturation(
+    themes_json_path: str,
+    embeddings_npy_path: str,
+    total_papers: int,
+) -> str:
+    """Compute saturation metrics per theme: coverage, coherence, balance.
+
+    Phase 4 — Saturation Review. DETERMINISTIC.
+
+    Every number in the output is computed by numpy — the LLM never
+    calculates these values. This eliminates hallucination risk for
+    percentages, scores, and ratios.
+
+    Metrics per theme:
+        coverage  = papers_in_theme / total_papers (exact percentage)
+        coherence = mean pairwise cosine similarity of member embeddings
+                    (1.0 = all identical, 0.0 = orthogonal)
+
+    Global metrics:
+        total_coverage  = papers in at least one theme / total_papers
+        balance_ratio   = largest_theme / smallest_theme
+        mean_coherence  = average of per-theme coherence scores
+
+    Args:
+        themes_json_path:    Path to themes.json.
+        embeddings_npy_path: Path to .emb.npy.
+        total_papers:        Total papers in corpus (from Phase 1 stats).
+
+    Returns:
+        JSON: per-theme metrics + global metrics. needs_review=True.
+    """
+    themes     = json.loads(Path(themes_json_path).read_text())
+    embeddings = np.load(embeddings_npy_path)
+
+    def _theme_metrics(t: dict) -> dict:
+        """Compute coverage and coherence for one theme."""
+        idx  = t.get("indices", [])
+        size = len(idx)
+        vecs = embeddings[idx or [0]]
+        sim  = cosine_similarity(vecs)
+        n    = len(vecs)
+        coherence = float(
+            (sim.sum() - n) / max(n * (n - 1), 1)
+        )
+        return {
+            "theme_id":    t.get("theme_id", 0),
+            "theme_label": t.get("theme_label", ""),
+            "papers":      size,
+            "coverage_pct": round(size / max(total_papers, 1) * 100, 2),
+            "coherence":    round(coherence, 4),
+        }
+
+    per_theme = list(map(_theme_metrics, themes))
+
+    all_paper_idx = set(sum(map(lambda t: t.get("indices", []), themes), []))
+    sizes         = list(map(lambda m: m["papers"], per_theme))
+    coherences    = list(map(lambda m: m["coherence"], per_theme))
+
+    global_metrics = {
+        "total_coverage_pct": round(len(all_paper_idx) / max(total_papers, 1) * 100, 2),
+        "balance_ratio":      round(max(sizes, default=1) / max(min(sizes, default=1), 1), 2),
+        "mean_coherence":     round(sum(coherences) / max(len(coherences), 1), 4),
+        "theme_count":        len(themes),
+    }
+
+    out_path = Path(themes_json_path).parent / "saturation.json"
+    result   = {"per_theme": per_theme, "global": global_metrics}
+    out_path.write_text(json.dumps(result, indent=2))
+
+    return json.dumps({
+        **global_metrics,
+        "per_theme":      per_theme,
+        "saturation_json": str(out_path),
+        "needs_review":    True,
+    }, indent=2)
+
+
+@tool
+def generate_theme_profiles(
+    themes_json_path: str,
+    embeddings_npy_path: str,
+    texts_parquet_path: str,
+    run_mode: str = "abstract",
+) -> str:
+    """Generate profile cards with top-5 nearest sentences per theme.
+
+    Phase 5 — Naming. DETERMINISTIC.
+
+    For each theme centroid, computes cosine similarity against ALL
+    embeddings and returns the 5 closest sentences. These are the
+    REAL sentences from the corpus — not generated, not recalled
+    from conversation history. The reviewer uses these to decide
+    on final theme names.
+
+    Steps:
+        1. Load themes.json with centroids
+        2. Load full embedding matrix
+        3. Load original texts from parquet
+        4. For each theme: cosine_similarity(centroid, all_embeddings)
+        5. Take top 5 by similarity score
+        6. Return exact sentence text + similarity score
+        7. Save profiles.json
+
+    Args:
+        themes_json_path:    Path to themes.json.
+        embeddings_npy_path: Path to .emb.npy.
+        texts_parquet_path:  Path to .clean.parquet (for original text).
+        run_mode:            'abstract' or 'title'.
+
+    Returns:
+        JSON: profiles list with top-5 sentences per theme. needs_review=True.
+    """
+    themes     = json.loads(Path(themes_json_path).read_text())
+    embeddings = np.load(embeddings_npy_path)
+    target     = f"{RUN_CONFIGS[run_mode][0]}_clean"
+    texts      = pd.read_parquet(texts_parquet_path)[target].tolist()
+
+    def _profile(t: dict) -> dict:
+        """Build a profile card for one theme: centroid → top 5 nearest."""
+        centroid = np.array(t["centroid"]).reshape(1, -1)
+        sims     = cosine_similarity(centroid, embeddings)[0]
+        top5_idx = np.argsort(sims)[::-1][:5].tolist()
+        top5     = list(map(
+            lambda i: {
+                "sentence_idx": i,
+                "text":         texts[i][:300],
+                "similarity":   round(float(sims[i]), 4),
+            },
+            top5_idx,
+        ))
+        return {
+            "theme_id":    t.get("theme_id", 0),
+            "theme_label": t.get("theme_label", ""),
+            "total_papers": t.get("total_papers", 0),
+            "top_5_sentences": top5,
+        }
+
+    profiles = list(map(_profile, themes))
+    out_path = Path(themes_json_path).parent / "profiles.json"
+    out_path.write_text(json.dumps(profiles, indent=2))
+
+    return json.dumps({
+        "profiles_count": len(profiles),
+        "profiles_json":  str(out_path),
+        "profiles":       profiles,
+        "needs_review":   True,
+    }, indent=2)
+
+
+@tool
+def compare_with_taxonomy(themes_json_path: str) -> str:
+    """Map each theme to PAJAIS 25 IS research categories via Mistral.
+
+    Phase 5.5 — PAJAIS Alignment. LLM-DEPENDENT (grounded in real labels).
+
+    Themes with alignment_score < 0.50 are flagged as potentially NOVEL.
+
+    Args:
+        themes_json_path: Path to themes.json.
+
+    Returns:
+        JSON: themes_aligned count + taxonomy_file path. needs_review=True.
+    """
+    themes = json.loads(Path(themes_json_path).read_text())
+
+    safe_themes = list(map(
+        lambda t: {k: v for k, v in t.items() if k not in ("centroid", "indices")},
+        themes,
+    ))
+
+    template = (
+        "You are an IS research taxonomy expert.\n\n"
+        "PAJAIS 25 Categories:\n{pajais}\n\n"
+        "Research themes:\n{themes}\n\n"
+        "For EACH theme return a JSON array where every element has:\n"
+        "  theme_label       : string\n"
+        "  pajais_categories : list of 1-3 matching PAJAIS category names\n"
+        "  alignment_score   : float 0.0-1.0\n"
+        "  notes             : one sentence justification\n\n"
+        "Return ONLY the JSON array — no markdown, no preamble."
+    )
+
+    result   = _mistral_chain(template).invoke({
+        "pajais": "\n".join(map(lambda c: f"- {c}", PAJAIS_25)),
+        "themes": json.dumps(safe_themes, indent=2),
+    })
+    out_path = Path(themes_json_path).parent / "taxonomy_alignment.json"
+    out_path.write_text(json.dumps(result, indent=2))
+
+    return json.dumps({
+        "themes_aligned": len(result),
+        "taxonomy_file":  str(out_path),
+        "needs_review":   True,
+    }, indent=2)
+
+
+@tool
+def generate_comparison_csv(
+    abstract_themes_path: str,
+    title_themes_path: str,
+    taxonomy_abstract_path: str,
+    taxonomy_title_path: str,
+) -> str:
+    """Build side-by-side abstract vs title comparison CSV.
+
+    Phase 6 — Report. DETERMINISTIC.
+
+    Joins on PAJAIS_Category. Delta_Score = Abstract - Title.
+
+    Args:
+        abstract_themes_path:   themes.json — abstract run.
+        title_themes_path:      themes.json — title run.
+        taxonomy_abstract_path: taxonomy_alignment.json — abstract run.
+        taxonomy_title_path:    taxonomy_alignment.json — title run.
+
+    Returns:
+        JSON: comparison_csv path, total_rows, columns. needs_review=True.
+    """
+    def _explode_taxonomy(path: str) -> pd.DataFrame:
+        """Flatten taxonomy alignment into one row per PAJAIS category."""
+        data = json.loads(Path(path).read_text())
+        rows = sum(
+            list(map(
+                lambda item: list(map(
+                    lambda cat: {
+                        "pajais_category": cat,
+                        "theme_label":     item.get("theme_label", ""),
+                        "alignment_score": item.get("alignment_score", 0.0),
+                    },
+                    item.get("pajais_categories", []),
+                )),
+                data,
+            )),
+            [],
+        )
+        return pd.DataFrame(rows)
+
+    df_abs   = _explode_taxonomy(taxonomy_abstract_path)
+    df_title = _explode_taxonomy(taxonomy_title_path)
+
+    df_abs.columns   = ["PAJAIS_Category", "Abstract_Theme",  "Abstract_Score"]
+    df_title.columns = ["PAJAIS_Category", "Title_Theme",     "Title_Score"]
+
+    merged = (
+        pd.merge(df_abs, df_title, on="PAJAIS_Category", how="outer")
+          .fillna({"Abstract_Score": 0.0, "Title_Score": 0.0,
+                   "Abstract_Theme": "",   "Title_Theme": ""})
+          .assign(Delta_Score=lambda d: (d["Abstract_Score"] - d["Title_Score"]).round(4))
+          .sort_values("PAJAIS_Category")
+          .reset_index(drop=True)
+    )
+
+    out_csv = Path(abstract_themes_path).parent / "abstract_vs_title_comparison.csv"
+    merged.to_csv(out_csv, index=False)
+
+    return json.dumps({
+        "comparison_csv": str(out_csv),
+        "total_rows":     len(merged),
+        "columns":        list(merged.columns),
+        "needs_review":   True,
+    }, indent=2)
+
+
+@tool
+def export_narrative(
+    taxonomy_alignment_path: str,
+    comparison_csv_path: str,
+    run_mode: str = "abstract",
+) -> str:
+    """Generate 500-word Section 7: Discussion & Implications via Mistral.
+
+    Phase 6 — Report. LLM-DEPENDENT (grounded in taxonomy + comparison data).
+
+    Args:
+        taxonomy_alignment_path: Path to taxonomy_alignment.json.
+        comparison_csv_path:     Path to comparison CSV.
+        run_mode:                'abstract' or 'title'.
+
+    Returns:
+        JSON: narrative_path, word_count, narrative text. needs_review=True.
+    """
+    alignment = json.loads(Path(taxonomy_alignment_path).read_text())
+
+    top_delta = (
+        pd.read_csv(comparison_csv_path)
+          .assign(_abs=lambda d: d["Delta_Score"].abs())
+          .sort_values("_abs", ascending=False)
+          .drop(columns=["_abs"])
+          .head(5)
+    )
+
+    template = (
+        "You are a senior IS researcher writing a systematic literature review.\n\n"
+        "Write Section 7: Discussion & Implications in exactly {word_count} words.\n\n"
+        "Run mode: {run_mode}\n\n"
+        "Taxonomy alignment (top 10):\n{alignment}\n\n"
+        "Top 5 divergent PAJAIS categories (abstract vs title):\n{divergence}\n\n"
+        "Requirements:\n"
+        "1. Discuss dominant themes and PAJAIS alignment.\n"
+        "2. Interpret divergence between abstract- and title-based models.\n"
+        "3. Highlight implications for IS research practice and future agenda.\n"
+        "4. Use formal academic register — no bullet points.\n"
+        "5. Return a JSON object with a single key 'narrative' containing the prose.\n\n"
+        "Return ONLY valid JSON."
+    )
+
+    result         = _mistral_chain(template).invoke({
+        "word_count": NARRATIVE_WORDS,
+        "run_mode":   run_mode,
+        "alignment":  json.dumps(alignment[:10], indent=2),
+        "divergence": top_delta.to_json(orient="records", indent=2),
+    })
+    narrative_text = result.get("narrative", str(result))
+    out_path       = Path(taxonomy_alignment_path).parent / "narrative.md"
+    out_path.write_text(
+        f"## Section 7: Discussion & Implications\n\n{narrative_text}\n",
+        encoding="utf-8",
+    )
+
+    return json.dumps({
+        "narrative_path": str(out_path),
+        "word_count":     len(narrative_text.split()),
+        "narrative":      narrative_text,
+        "needs_review":   True,
+    }, indent=2)
+
+
+ALL_TOOLS = [
+    load_scopus_csv,
+    run_bertopic_discovery,
+    label_topics_with_llm,
+    reassign_sentences,
+    consolidate_into_themes,
+    compute_saturation,
+    generate_theme_profiles,
+    compare_with_taxonomy,
+    generate_comparison_csv,
+    export_narrative,
+]