tools.py

"""
tools.py — 10 @tool functions for BERTopic-style topic modelling.

Pipeline (called in this order by the LLM agent):

    1.  load_scopus_csv          — ingest CSV, strip boilerplate, save .parquet
    2.  run_bertopic_discovery   — embed → cosine agglomerative cluster (min 3
                                    members) → centroids → orphan report → 4 charts
    3.  label_topics_with_llm    — Mistral labels top 100 clusters
    4.  reassign_sentences       — move orphan/misplaced sentences between clusters
    5.  consolidate_into_themes  — merge reviewer-approved groups
    6.  compute_saturation       — coverage %, coherence, balance per theme
    7.  generate_theme_profiles  — top 5 nearest sentences per theme centroid
    8.  compare_with_taxonomy    — map themes to PAJAIS 25 categories
    9.  generate_comparison_csv  — abstract vs title side-by-side
    10. export_narrative         — 500-word Section 7 via Mistral

Design rules:

    Every number, percentage, score, or list of sentences presented to the
    reviewer MUST come from a tool — never from the LLM's imagination.

    Deterministic tools (1,2,4,5,6,7,9): same input → same output, every run.
    LLM-dependent tools (3,8,10): grounded in real data passed via prompt,
    but labels/mappings/narrative may vary slightly between runs.
    All LLM-dependent outputs require reviewer approval before advancing.

    ZERO if/elif/else — all decisions by the LLM
    ZERO for/while    — list(map(...)) and numpy vectorised ops
    ZERO try/except   — errors surface to the LLM via ToolNode

Constants reference:

    EMBED_MODEL = "all-MiniLM-L6-v2"
        384d sentence embeddings. Runs locally, no API calls.
        normalize_embeddings=True → cosine similarity = dot product.

    CLUSTER_THRESHOLD = 0.7
        Agglomerative stops merging at cosine distance > 0.7.
        0.5 → ~2,102 clusters. 0.7 → ~100 clusters. 0.8 → ~30 clusters.

    MIN_CLUSTER_SIZE = 3
        Clusters with fewer than 3 members are dissolved. Their sentences
        become orphans (label=-1) reported to the reviewer for reassignment.

    N_CENTROIDS = 5
        Top clusters extracted for initial discovery report and charts.

    TOP_TOPICS_LLM = 100
        Maximum clusters sent to Mistral for labelling.

    NARRATIVE_WORDS = 500
        Target word count for Section 7 narrative.

    PAJAIS_25
        25 IS research categories from Jiang et al. (2019).
        Used in Phase 5.5 for taxonomy alignment.

    BOILERPLATE_PATTERNS (9 regexes)
        Strip publisher noise: copyright, DOI, Elsevier, Springer,
        IEEE, Wiley, Taylor & Francis.
"""

from __future__ import annotations

import json
import re
import numpy as np
import pandas as pd
import plotly.graph_objects as go

from pathlib import Path
from langchain_core.tools import tool
from langchain_mistralai import ChatMistralAI
from langchain_core.prompts import PromptTemplate
from langchain_core.output_parsers import JsonOutputParser
from sentence_transformers import SentenceTransformer
from sklearn.cluster import AgglomerativeClustering
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.preprocessing import normalize
from sklearn.decomposition import PCA


RUN_CONFIGS = {
    "abstract": ["Abstract"],
    "title":    ["Title"],
}

PAJAIS_25 = [
    "Accounting Information Systems",
    "Artificial Intelligence & Expert Systems",
    "Big Data & Analytics",
    "Business Intelligence & Decision Support",
    "Cloud Computing",
    "Cybersecurity & Privacy",
    "Database Management",
    "Digital Transformation",
    "E-Business & E-Commerce",
    "Enterprise Resource Planning",
    "Fintech & Digital Finance",
    "Geographic Information Systems",
    "Health Informatics",
    "Human-Computer Interaction",
    "Information Systems Development",
    "IT Governance & Management",
    "IT Strategy & Competitive Advantage",
    "Knowledge Management",
    "Machine Learning & Deep Learning",
    "Mobile Computing",
    "Natural Language Processing",
    "Recommender Systems",
    "Social Media & Web 2.0",
    "Supply Chain & Logistics IS",
    "Virtual Reality & Augmented Reality",
]

BOILERPLATE_PATTERNS = [
    r"©\s*\d{4}",
    r"all rights reserved",
    r"published by elsevier",
    r"this article is protected",
    r"doi:\s*10\.\d{4,}",
    r"springer nature",
    r"ieee xplore",
    r"wiley online library",
    r"taylor & francis",
]

BOILERPLATE_RE    = re.compile("|".join(BOILERPLATE_PATTERNS), flags=re.IGNORECASE)
SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+")
EMBED_MODEL       = "all-MiniLM-L6-v2"
N_CENTROIDS       = 5
CLUSTER_THRESHOLD = 0.7
MIN_CLUSTER_SIZE  = 3
TOP_TOPICS_LLM    = 100
NARRATIVE_WORDS   = 500


def _clean_text(text: str) -> str:
    """Remove publisher boilerplate from a single text string.

    Applies 9-pattern BOILERPLATE_RE regex to strip copyright notices,
    DOI prefixes, and publisher tags that would pollute embeddings.

    Args:
        text: Raw abstract or title string.

    Returns:
        Cleaned string with boilerplate removed and whitespace trimmed.
    """
    return BOILERPLATE_RE.sub("", str(text)).strip()


def _sentence_count(text: str) -> int:
    """Count sentences using regex split on terminal punctuation.

    Args:
        text: Cleaned abstract or title text.

    Returns:
        Number of sentences (minimum 1 for any non-empty input).
    """
    return len(SENTENCE_SPLIT_RE.split(text.strip()))


def _embed(texts: list[str]) -> np.ndarray:
    """Embed texts into 384d L2-normalized unit vectors.

    Uses SentenceTransformer('all-MiniLM-L6-v2') locally — no API calls.
    normalize_embeddings=True ensures cosine_similarity = dot product.

    Args:
        texts: List of N cleaned text strings.

    Returns:
        np.ndarray shape (N, 384), dtype float32, L2-normalized.
    """
    model = SentenceTransformer(EMBED_MODEL)
    raw   = model.encode(texts, show_progress_bar=False, normalize_embeddings=True)
    return np.array(raw, dtype=np.float32)


def _cosine_cluster(matrix: np.ndarray, threshold: float, min_size: int) -> np.ndarray:
    """Cluster embeddings using agglomerative cosine clustering.

    Works DIRECTLY in 384d space — no UMAP. After clustering, any cluster
    with fewer than min_size members is dissolved: its sentences get
    label=-1 (orphan) and are reported to the reviewer for reassignment.

    Algorithm:
        1. Start: every text is its own cluster.
        2. Merge the two closest clusters (average cosine distance).
        3. Repeat until smallest distance exceeds threshold.
        4. Post-process: dissolve clusters smaller than min_size.

    Args:
        matrix:    (N, 384) embedding matrix, L2-normalized.
        threshold: Max cosine distance for merging (0.7 → ~100 clusters).
        min_size:  Minimum members per cluster (3). Smaller → orphan.

    Returns:
        np.ndarray shape (N,) with integer labels. -1 = orphan.
    """
    normed = normalize(matrix, norm="l2")
    model  = AgglomerativeClustering(
        n_clusters=None,
        metric="cosine",
        linkage="average",
        distance_threshold=threshold,
    )
    labels = model.fit_predict(normed).astype(int)
    unique, counts = np.unique(labels, return_counts=True)
    small_clusters = unique[counts < min_size]
    return np.where(np.isin(labels, small_clusters), -1, labels)


def _centroid(vecs: np.ndarray) -> np.ndarray:
    """Compute L2-normalized centroid (average direction in 384d space).

    Args:
        vecs: (M, 384) matrix of member embeddings for one cluster.

    Returns:
        1d np.ndarray shape (384,), L2-normalized.
    """
    return normalize(vecs.mean(axis=0, keepdims=True), norm="l2")[0]


def _top_n_centroids(matrix: np.ndarray, labels: np.ndarray, n: int) -> list[dict]:
    """Extract N largest clusters by size and compute their centroids.

    Excludes orphans (label=-1) from the ranking.

    Args:
        matrix: (N, 384) full embedding matrix.
        labels: (N,) integer cluster labels (-1 = orphan).
        n:      How many top clusters to return.

    Returns:
        List of N dicts with: label, size, indices, centroid.
    """
    valid_mask = labels >= 0
    valid_labels = labels[valid_mask]
    unique, counts = np.unique(valid_labels, return_counts=True)
    order      = np.argsort(counts)[::-1][:n]
    top_labels = unique[order]

    def _build(lbl: int) -> dict:
        """Build summary dict for one cluster."""
        idx = np.where(labels == lbl)[0].tolist()
        return {
            "label":    int(lbl),
            "size":     len(idx),
            "indices":  idx,
            "centroid": _centroid(matrix[idx]),
        }

    return list(map(_build, top_labels))


def _mistral_chain(template_str: str):
    """Create PromptTemplate → ChatMistralAI → JsonOutputParser chain.

    Args:
        template_str: Prompt template with {variable} placeholders.

    Returns:
        LangChain Runnable chain that accepts dict and returns parsed JSON.
    """
    llm    = ChatMistralAI(model="mistral-large-latest", temperature=0)
    prompt = PromptTemplate.from_template(template_str)
    return prompt | llm | JsonOutputParser()


def _dark_layout(title: str) -> dict:
    """Return Plotly layout dict with dark theme styling.

    Args:
        title: Chart title string.

    Returns:
        Dict for fig.update_layout(**_dark_layout("...")).
    """
    return dict(
        title=title, paper_bgcolor="#0F172A", plot_bgcolor="#0F172A",
        font=dict(color="#CBD5E1", family="Sora,sans-serif"),
        margin=dict(t=50, b=40, l=40, r=20),
    )


@tool
def load_scopus_csv(csv_path: str, run_mode: str = "abstract") -> str:
    """Load a Scopus CSV, count papers/sentences, apply boilerplate filter.

    Phase 1 — Familiarisation. DETERMINISTIC.

    Steps:
        1. Read CSV, drop rows where target column is null
        2. Apply 9-pattern boilerplate regex to clean each text
        3. Count sentences per paper
        4. Save cleaned DataFrame as .parquet

    Args:
        csv_path: Path to raw Scopus CSV.
        run_mode: 'abstract' or 'title'.

    Returns:
        JSON: total_papers, total_sentences, columns_used,
        boilerplate_removed, cleaned_parquet, run_mode.
    """
    cols   = RUN_CONFIGS[run_mode]
    target = cols[0]

    df            = pd.read_csv(csv_path).dropna(subset=[target]).reset_index(drop=True)
    raw_texts     = df[target].tolist()
    cleaned_texts = list(map(_clean_text, raw_texts))

    boilerplate_removed = sum(map(
        lambda pair: int(pair[0] != pair[1]),
        zip(raw_texts, cleaned_texts),
    ))

    df[f"{target}_clean"] = cleaned_texts
    df["sentence_count"]  = list(map(_sentence_count, cleaned_texts))

    out_path = Path(csv_path).with_suffix(".clean.parquet")
    df.to_parquet(out_path, index=False)

    return json.dumps({
        "total_papers":        len(df),
        "total_sentences":     int(df["sentence_count"].sum()),
        "columns_used":        cols,
        "boilerplate_removed": boilerplate_removed,
        "cleaned_parquet":     str(out_path),
        "run_mode":            run_mode,
    }, indent=2)


@tool
def run_bertopic_discovery(parquet_path: str, run_mode: str = "abstract") -> str:
    """Embed texts, cluster them, report orphans, generate charts.

    Phase 2 — Initial Codes. DETERMINISTIC.

    Steps:
        1. Load cleaned parquet, drop Author Keywords columns (RULE 8)
        2. Embed all texts → N x 384 matrix of unit vectors
        3. Save embedding matrix as .emb.npy
        4. Cluster in 384d space (NO UMAP), min 3 members per cluster
        5. Sentences in clusters < 3 members become orphans (label=-1)
        6. Extract top-5 clusters, compute centroids
        7. Save summaries.json with clusters + orphan list
        8. Generate 4 Plotly HTML charts

    Args:
        parquet_path: Path to .clean.parquet from load_scopus_csv.
        run_mode:     'abstract' or 'title'.

    Returns:
        JSON: total_clusters, orphan_count, summaries_json, embeddings_npy,
        charts dict.
    """
    cols   = RUN_CONFIGS[run_mode]
    target = f"{cols[0]}_clean"

    df = pd.read_parquet(parquet_path).drop(
        columns=[c for c in pd.read_parquet(parquet_path).columns
                 if re.search(r"keyword|author", c, re.I)],
        errors="ignore",
    )

    texts      = df[target].tolist()
    embeddings = _embed(texts)
    base       = Path(parquet_path).parent

    np.save(str(base / Path(parquet_path).stem) + ".emb.npy", embeddings)

    labels        = _cosine_cluster(embeddings, CLUSTER_THRESHOLD, MIN_CLUSTER_SIZE)
    orphan_idx    = np.where(labels == -1)[0].tolist()
    orphan_count  = len(orphan_idx)
    valid_count   = int((labels >= 0).sum())
    n_clusters    = int(np.unique(labels[labels >= 0]).shape[0])
    top_centroids = _top_n_centroids(embeddings, labels, N_CENTROIDS)

    def _topic_row(tc: dict) -> dict:
        """Convert centroid dict into summary row for summaries.json."""
        return {
            "topic_id":       tc["label"],
            "size":           tc["size"],
            "representative": texts[tc["indices"][0]][:200],
            "indices":        tc["indices"],
        }

    summaries = list(map(_topic_row, top_centroids))

    orphans = list(map(
        lambda i: {"sentence_idx": int(i), "text": texts[i][:200]},
        orphan_idx,
    ))

    output = {"clusters": summaries, "orphans": orphans}
    (base / "summaries.json").write_text(json.dumps(output, indent=2))

    unique, counts = np.unique(labels[labels >= 0], return_counts=True)
    order          = np.argsort(counts)[::-1][:20]
    c1 = go.Figure(go.Bar(
        x=list(map(str, unique[order])), y=counts[order].tolist(),
        marker_color="#3B82F6", text=counts[order].tolist(), textposition="outside",
    ))
    c1.update_layout(**_dark_layout("Topic Size Distribution (Top 20)"),
                     xaxis=dict(showgrid=False),
                     yaxis=dict(showgrid=True, gridcolor="#1E293B"))
    c1.write_html(str(base / "chart_topic_sizes.html"))

    centroid_matrix = np.vstack([tc["centroid"] for tc in top_centroids])
    sim_matrix      = cosine_similarity(centroid_matrix)
    clabels         = list(map(lambda tc: f"T{tc['label']}", top_centroids))
    c2 = go.Figure(go.Heatmap(z=sim_matrix, x=clabels, y=clabels, colorscale="Blues"))
    c2.update_layout(**_dark_layout("Top-5 Centroid Cosine Similarity"))
    c2.write_html(str(base / "chart_centroid_heatmap.html"))

    sc = df.get("sentence_count", pd.Series([0] * len(df))).tolist()
    c3 = go.Figure(go.Histogram(x=sc, nbinsx=40, marker_color="#22D3EE"))
    c3.update_layout(**_dark_layout("Sentence Count Distribution"),
                     xaxis=dict(showgrid=False),
                     yaxis=dict(showgrid=True, gridcolor="#1E293B"))
    c3.write_html(str(base / "chart_sentence_distribution.html"))

    coords     = PCA(n_components=2).fit_transform(centroid_matrix)
    point_text = list(map(lambda tc: f"T{tc['label']}({tc['size']})", top_centroids))
    c4 = go.Figure(go.Scatter(
        x=coords[:, 0].tolist(), y=coords[:, 1].tolist(),
        mode="markers+text", text=point_text, textposition="top center",
        marker=dict(size=12, color="#F59E0B", line=dict(width=1, color="#0F172A")),
    ))
    c4.update_layout(**_dark_layout("Top-5 Centroids — PCA Projection"))
    c4.write_html(str(base / "chart_centroid_pca.html"))

    emb_path = str(base / Path(parquet_path).stem) + ".emb.npy"
    return json.dumps({
        "total_clusters":  n_clusters,
        "orphan_count":    orphan_count,
        "valid_sentences": valid_count,
        "top_centroids":   N_CENTROIDS,
        "summaries_json":  str(base / "summaries.json"),
        "embeddings_npy":  emb_path,
        "needs_review":    True,
        "charts": {
            "topic_sizes":      str(base / "chart_topic_sizes.html"),
            "centroid_heatmap": str(base / "chart_centroid_heatmap.html"),
            "sentence_dist":    str(base / "chart_sentence_distribution.html"),
            "centroid_pca":     str(base / "chart_centroid_pca.html"),
        },
    }, indent=2)


@tool
def label_topics_with_llm(summaries_json_path: str) -> str:
    """Send top-100 topic summaries to Mistral for labelling.

    Phase 2 — Labelling. LLM-DEPENDENT (grounded in real sentences).

    Steps:
        1. Load summaries.json clusters (not orphans)
        2. Take top 100 by size
        3. Mistral reads representative sentences → assigns labels
        4. Returns: topic_id, label, rationale, confidence per cluster
        5. Save as topic_labels.json

    Args:
        summaries_json_path: Path to summaries.json.

    Returns:
        JSON: labelled_topics count + output path. needs_review=True.
    """
    data          = json.loads(Path(summaries_json_path).read_text())
    summaries     = data.get("clusters", data)[:TOP_TOPICS_LLM]

    template = (
        "You are a scientific topic labelling expert.\n\n"
        "Below are {n} topic summaries from a BERTopic analysis of academic papers.\n"
        "Each summary has: topic_id, size, representative text.\n\n"
        "{summaries}\n\n"
        "For EACH topic return a JSON array where every element has:\n"
        "  topic_id   : integer (copy from input)\n"
        "  label      : 2-5 word snake_case topic label\n"
        "  rationale  : one sentence justification\n"
        "  confidence : float 0.0-1.0\n\n"
        "Return ONLY the JSON array — no markdown, no preamble."
    )

    result   = _mistral_chain(template).invoke({
        "n":         len(summaries),
        "summaries": json.dumps(summaries, indent=2),
    })
    out_path = Path(summaries_json_path).parent / "topic_labels.json"
    out_path.write_text(json.dumps(result, indent=2))

    return json.dumps({
        "labelled_topics": len(result),
        "output":          str(out_path),
        "needs_review":    True,
    }, indent=2)


@tool
def reassign_sentences(
    summaries_json_path: str,
    embeddings_npy_path: str,
    move_instructions: str,
) -> str:
    """Move orphan or misplaced sentences between clusters.

    Phase 2 — Orphan handling. DETERMINISTIC.

    The reviewer specifies moves as JSON:
        [{"sentence_idx": 42, "to_cluster": 3},
         {"sentence_idx": 99, "to_cluster": "new"}]

    For "new" targets, a fresh cluster ID is assigned.
    After all moves, centroids are recomputed for affected clusters.

    Steps:
        1. Load summaries.json and embeddings
        2. Parse move instructions
        3. Update cluster assignments
        4. Recompute centroids for affected clusters
        5. Save updated summaries.json

    Args:
        summaries_json_path: Path to summaries.json.
        embeddings_npy_path: Path to .emb.npy.
        move_instructions:   JSON array of {sentence_idx, to_cluster} dicts.

    Returns:
        JSON: moves_applied count, orphans_remaining, updated summaries path.
    """
    data       = json.loads(Path(summaries_json_path).read_text())
    embeddings = np.load(embeddings_npy_path)
    moves      = json.loads(move_instructions)
    clusters   = data.get("clusters", [])
    orphans    = data.get("orphans", [])

    all_indices = {}
    list(map(
        lambda c: all_indices.update({idx: c["topic_id"] for idx in c.get("indices", [])}),
        clusters,
    ))

    max_id = max(map(lambda c: c.get("topic_id", 0), clusters), default=0)
    new_id_counter = [max_id + 1]

    def _apply_move(m: dict) -> dict:
        """Apply one move instruction, return the resolved target cluster ID."""
        s_idx = m["sentence_idx"]
        target = m["to_cluster"]
        resolved = (target == "new") and new_id_counter.__setitem__(0, new_id_counter[0] + 1) or target
        final_id = new_id_counter[0] - 1 * (target == "new") + target * (target != "new")
        all_indices[s_idx] = int(target) * (target != "new") + new_id_counter[0] * (target == "new")
        return {"sentence_idx": s_idx, "assigned_to": all_indices[s_idx]}

    applied = list(map(_apply_move, moves))

    unique_clusters = set(all_indices.values())

    def _rebuild_cluster(cid: int) -> dict:
        """Rebuild a cluster dict from the updated index map."""
        idx = [k for k, v in all_indices.items() if v == cid]
        vecs = embeddings[idx or [0]]
        return {
            "topic_id":       int(cid),
            "size":           len(idx),
            "representative": "",
            "indices":        idx,
            "centroid":       _centroid(vecs).tolist(),
        }

    updated_clusters = list(map(_rebuild_cluster, sorted(unique_clusters)))
    remaining_orphan_idx = [o["sentence_idx"] for o in orphans
                           if o["sentence_idx"] not in all_indices]

    output = {
        "clusters": updated_clusters,
        "orphans":  list(map(
            lambda i: {"sentence_idx": i, "text": ""},
            remaining_orphan_idx,
        )),
    }
    Path(summaries_json_path).write_text(json.dumps(output, indent=2))

    return json.dumps({
        "moves_applied":     len(applied),
        "orphans_remaining": len(remaining_orphan_idx),
        "summaries_json":    summaries_json_path,
        "needs_review":      True,
    }, indent=2)


@tool
def consolidate_into_themes(
    labels_json_path: str,
    embeddings_npy_path: str,
    approved_topic_ids: str,
) -> str:
    """Merge approved topic groups into consolidated themes.

    Phase 3 — Theme Search. DETERMINISTIC.

    Steps:
        1. Load topic_labels.json and embedding matrix
        2. Parse approved groupings (JSON array of arrays)
        3. Pool all member embeddings per group
        4. Compute fresh L2-normalized centroid per merged group
        5. Build theme name from joined sub-labels
        6. Save themes.json

    Args:
        labels_json_path:    Path to topic_labels.json.
        embeddings_npy_path: Path to .emb.npy.
        approved_topic_ids:  JSON array of arrays, e.g. [[0,1,2],[3,4],[5]].

    Returns:
        JSON: themes_created count + themes_json path. needs_review=True.
    """
    labels_data = json.loads(Path(labels_json_path).read_text())
    embeddings  = np.load(embeddings_npy_path)
    groups      = json.loads(approved_topic_ids)
    label_map   = {item["topic_id"]: item for item in labels_data}

    def _merge_group(group_ids: list[int]) -> dict:
        """Merge topic IDs into one theme, recompute centroid."""
        members    = [m for m in map(label_map.get, group_ids) if m is not None]
        all_idx    = sum(map(lambda m: m.get("indices", []), members), [])
        vecs       = embeddings[all_idx or [0]]
        centroid   = _centroid(vecs)
        sub_labels = list(map(lambda m: m.get("label", ""), members))
        theme_name = "_".join(
            dict.fromkeys(sum(map(lambda lbl: lbl.split("_"), sub_labels), []))
        )[:60]
        return {
            "theme_id":     group_ids[0],
            "theme_label":  theme_name,
            "merged_ids":   group_ids,
            "total_papers": len(set(all_idx)),
            "indices":      all_idx,
            "centroid":     centroid.tolist(),
        }

    themes   = list(map(_merge_group, groups))
    out_path = Path(labels_json_path).parent / "themes.json"
    out_path.write_text(json.dumps(themes, indent=2))

    return json.dumps({
        "themes_created": len(themes),
        "themes_json":    str(out_path),
        "needs_review":   True,
    }, indent=2)


@tool
def compute_saturation(
    themes_json_path: str,
    embeddings_npy_path: str,
    total_papers: int,
) -> str:
    """Compute saturation metrics per theme: coverage, coherence, balance.

    Phase 4 — Saturation Review. DETERMINISTIC.

    Every number in the output is computed by numpy — the LLM never
    calculates these values. This eliminates hallucination risk for
    percentages, scores, and ratios.

    Metrics per theme:
        coverage  = papers_in_theme / total_papers (exact percentage)
        coherence = mean pairwise cosine similarity of member embeddings
                    (1.0 = all identical, 0.0 = orthogonal)

    Global metrics:
        total_coverage  = papers in at least one theme / total_papers
        balance_ratio   = largest_theme / smallest_theme
        mean_coherence  = average of per-theme coherence scores

    Args:
        themes_json_path:    Path to themes.json.
        embeddings_npy_path: Path to .emb.npy.
        total_papers:        Total papers in corpus (from Phase 1 stats).

    Returns:
        JSON: per-theme metrics + global metrics. needs_review=True.
    """
    themes     = json.loads(Path(themes_json_path).read_text())
    embeddings = np.load(embeddings_npy_path)

    def _theme_metrics(t: dict) -> dict:
        """Compute coverage and coherence for one theme."""
        idx  = t.get("indices", [])
        size = len(idx)
        vecs = embeddings[idx or [0]]
        sim  = cosine_similarity(vecs)
        n    = len(vecs)
        coherence = float(
            (sim.sum() - n) / max(n * (n - 1), 1)
        )
        return {
            "theme_id":    t.get("theme_id", 0),
            "theme_label": t.get("theme_label", ""),
            "papers":      size,
            "coverage_pct": round(size / max(total_papers, 1) * 100, 2),
            "coherence":    round(coherence, 4),
        }

    per_theme = list(map(_theme_metrics, themes))

    all_paper_idx = set(sum(map(lambda t: t.get("indices", []), themes), []))
    sizes         = list(map(lambda m: m["papers"], per_theme))
    coherences    = list(map(lambda m: m["coherence"], per_theme))

    global_metrics = {
        "total_coverage_pct": round(len(all_paper_idx) / max(total_papers, 1) * 100, 2),
        "balance_ratio":      round(max(sizes, default=1) / max(min(sizes, default=1), 1), 2),
        "mean_coherence":     round(sum(coherences) / max(len(coherences), 1), 4),
        "theme_count":        len(themes),
    }

    out_path = Path(themes_json_path).parent / "saturation.json"
    result   = {"per_theme": per_theme, "global": global_metrics}
    out_path.write_text(json.dumps(result, indent=2))

    return json.dumps({
        **global_metrics,
        "per_theme":      per_theme,
        "saturation_json": str(out_path),
        "needs_review":    True,
    }, indent=2)


@tool
def generate_theme_profiles(
    themes_json_path: str,
    embeddings_npy_path: str,
    texts_parquet_path: str,
    run_mode: str = "abstract",
) -> str:
    """Generate profile cards with top-5 nearest sentences per theme.

    Phase 5 — Naming. DETERMINISTIC.

    For each theme centroid, computes cosine similarity against ALL
    embeddings and returns the 5 closest sentences. These are the
    REAL sentences from the corpus — not generated, not recalled
    from conversation history. The reviewer uses these to decide
    on final theme names.

    Steps:
        1. Load themes.json with centroids
        2. Load full embedding matrix
        3. Load original texts from parquet
        4. For each theme: cosine_similarity(centroid, all_embeddings)
        5. Take top 5 by similarity score
        6. Return exact sentence text + similarity score
        7. Save profiles.json

    Args:
        themes_json_path:    Path to themes.json.
        embeddings_npy_path: Path to .emb.npy.
        texts_parquet_path:  Path to .clean.parquet (for original text).
        run_mode:            'abstract' or 'title'.

    Returns:
        JSON: profiles list with top-5 sentences per theme. needs_review=True.
    """
    themes     = json.loads(Path(themes_json_path).read_text())
    embeddings = np.load(embeddings_npy_path)
    target     = f"{RUN_CONFIGS[run_mode][0]}_clean"
    texts      = pd.read_parquet(texts_parquet_path)[target].tolist()

    def _profile(t: dict) -> dict:
        """Build a profile card for one theme: centroid → top 5 nearest."""
        centroid = np.array(t["centroid"]).reshape(1, -1)
        sims     = cosine_similarity(centroid, embeddings)[0]
        top5_idx = np.argsort(sims)[::-1][:5].tolist()
        top5     = list(map(
            lambda i: {
                "sentence_idx": i,
                "text":         texts[i][:300],
                "similarity":   round(float(sims[i]), 4),
            },
            top5_idx,
        ))
        return {
            "theme_id":    t.get("theme_id", 0),
            "theme_label": t.get("theme_label", ""),
            "total_papers": t.get("total_papers", 0),
            "top_5_sentences": top5,
        }

    profiles = list(map(_profile, themes))
    out_path = Path(themes_json_path).parent / "profiles.json"
    out_path.write_text(json.dumps(profiles, indent=2))

    return json.dumps({
        "profiles_count": len(profiles),
        "profiles_json":  str(out_path),
        "profiles":       profiles,
        "needs_review":   True,
    }, indent=2)


@tool
def compare_with_taxonomy(themes_json_path: str) -> str:
    """Map each theme to PAJAIS 25 IS research categories via Mistral.

    Phase 5.5 — PAJAIS Alignment. LLM-DEPENDENT (grounded in real labels).

    Themes with alignment_score < 0.50 are flagged as potentially NOVEL.

    Args:
        themes_json_path: Path to themes.json.

    Returns:
        JSON: themes_aligned count + taxonomy_file path. needs_review=True.
    """
    themes = json.loads(Path(themes_json_path).read_text())

    safe_themes = list(map(
        lambda t: {k: v for k, v in t.items() if k not in ("centroid", "indices")},
        themes,
    ))

    template = (
        "You are an IS research taxonomy expert.\n\n"
        "PAJAIS 25 Categories:\n{pajais}\n\n"
        "Research themes:\n{themes}\n\n"
        "For EACH theme return a JSON array where every element has:\n"
        "  theme_label       : string\n"
        "  pajais_categories : list of 1-3 matching PAJAIS category names\n"
        "  alignment_score   : float 0.0-1.0\n"
        "  notes             : one sentence justification\n\n"
        "Return ONLY the JSON array — no markdown, no preamble."
    )

    result   = _mistral_chain(template).invoke({
        "pajais": "\n".join(map(lambda c: f"- {c}", PAJAIS_25)),
        "themes": json.dumps(safe_themes, indent=2),
    })
    out_path = Path(themes_json_path).parent / "taxonomy_alignment.json"
    out_path.write_text(json.dumps(result, indent=2))

    return json.dumps({
        "themes_aligned": len(result),
        "taxonomy_file":  str(out_path),
        "needs_review":   True,
    }, indent=2)


@tool
def generate_comparison_csv(
    abstract_themes_path: str,
    title_themes_path: str,
    taxonomy_abstract_path: str,
    taxonomy_title_path: str,
) -> str:
    """Build side-by-side abstract vs title comparison CSV.

    Phase 6 — Report. DETERMINISTIC.

    Joins on PAJAIS_Category. Delta_Score = Abstract - Title.

    Args:
        abstract_themes_path:   themes.json — abstract run.
        title_themes_path:      themes.json — title run.
        taxonomy_abstract_path: taxonomy_alignment.json — abstract run.
        taxonomy_title_path:    taxonomy_alignment.json — title run.

    Returns:
        JSON: comparison_csv path, total_rows, columns. needs_review=True.
    """
    def _explode_taxonomy(path: str) -> pd.DataFrame:
        """Flatten taxonomy alignment into one row per PAJAIS category."""
        data = json.loads(Path(path).read_text())
        rows = sum(
            list(map(
                lambda item: list(map(
                    lambda cat: {
                        "pajais_category": cat,
                        "theme_label":     item.get("theme_label", ""),
                        "alignment_score": item.get("alignment_score", 0.0),
                    },
                    item.get("pajais_categories", []),
                )),
                data,
            )),
            [],
        )
        return pd.DataFrame(rows)

    df_abs   = _explode_taxonomy(taxonomy_abstract_path)
    df_title = _explode_taxonomy(taxonomy_title_path)

    df_abs.columns   = ["PAJAIS_Category", "Abstract_Theme",  "Abstract_Score"]
    df_title.columns = ["PAJAIS_Category", "Title_Theme",     "Title_Score"]

    merged = (
        pd.merge(df_abs, df_title, on="PAJAIS_Category", how="outer")
          .fillna({"Abstract_Score": 0.0, "Title_Score": 0.0,
                   "Abstract_Theme": "",   "Title_Theme": ""})
          .assign(Delta_Score=lambda d: (d["Abstract_Score"] - d["Title_Score"]).round(4))
          .sort_values("PAJAIS_Category")
          .reset_index(drop=True)
    )

    out_csv = Path(abstract_themes_path).parent / "abstract_vs_title_comparison.csv"
    merged.to_csv(out_csv, index=False)

    return json.dumps({
        "comparison_csv": str(out_csv),
        "total_rows":     len(merged),
        "columns":        list(merged.columns),
        "needs_review":   True,
    }, indent=2)


@tool
def export_narrative(
    taxonomy_alignment_path: str,
    comparison_csv_path: str,
    run_mode: str = "abstract",
) -> str:
    """Generate 500-word Section 7: Discussion & Implications via Mistral.

    Phase 6 — Report. LLM-DEPENDENT (grounded in taxonomy + comparison data).

    Args:
        taxonomy_alignment_path: Path to taxonomy_alignment.json.
        comparison_csv_path:     Path to comparison CSV.
        run_mode:                'abstract' or 'title'.

    Returns:
        JSON: narrative_path, word_count, narrative text. needs_review=True.
    """
    alignment = json.loads(Path(taxonomy_alignment_path).read_text())

    top_delta = (
        pd.read_csv(comparison_csv_path)
          .assign(_abs=lambda d: d["Delta_Score"].abs())
          .sort_values("_abs", ascending=False)
          .drop(columns=["_abs"])
          .head(5)
    )

    template = (
        "You are a senior IS researcher writing a systematic literature review.\n\n"
        "Write Section 7: Discussion & Implications in exactly {word_count} words.\n\n"
        "Run mode: {run_mode}\n\n"
        "Taxonomy alignment (top 10):\n{alignment}\n\n"
        "Top 5 divergent PAJAIS categories (abstract vs title):\n{divergence}\n\n"
        "Requirements:\n"
        "1. Discuss dominant themes and PAJAIS alignment.\n"
        "2. Interpret divergence between abstract- and title-based models.\n"
        "3. Highlight implications for IS research practice and future agenda.\n"
        "4. Use formal academic register — no bullet points.\n"
        "5. Return a JSON object with a single key 'narrative' containing the prose.\n\n"
        "Return ONLY valid JSON."
    )

    result         = _mistral_chain(template).invoke({
        "word_count": NARRATIVE_WORDS,
        "run_mode":   run_mode,
        "alignment":  json.dumps(alignment[:10], indent=2),
        "divergence": top_delta.to_json(orient="records", indent=2),
    })
    narrative_text = result.get("narrative", str(result))
    out_path       = Path(taxonomy_alignment_path).parent / "narrative.md"
    out_path.write_text(
        f"## Section 7: Discussion & Implications\n\n{narrative_text}\n",
        encoding="utf-8",
    )

    return json.dumps({
        "narrative_path": str(out_path),
        "word_count":     len(narrative_text.split()),
        "narrative":      narrative_text,
        "needs_review":   True,
    }, indent=2)


ALL_TOOLS = [
    load_scopus_csv,
    run_bertopic_discovery,
    label_topics_with_llm,
    reassign_sentences,
    consolidate_into_themes,
    compute_saturation,
    generate_theme_profiles,
    compare_with_taxonomy,
    generate_comparison_csv,
    export_narrative,
]