Delete tools.py

tools.py

@@ -1,623 +0,0 @@
-"""tools.py — Sentence-level BERTopic pipeline + Mistral LLM. Version 3.0.0 | 4 April 2026. ZERO for/while/if.
-
-PIPELINE:
-  Paper → split into sentences → each sentence gets paper_id + sent_id + metadata
-  → embed sentences (384d) → AgglomerativeClustering cosine → centroid nearest 5 sentences
-  → Mistral labels topics from sentence evidence + paper metadata
-  → one paper can span MULTIPLE topics
-"""
-from langchain_core.tools import tool
-import os
-import json
-import re
-import numpy as np
-import pandas as pd
-
-# ═══════════════════════════════════════════════
-# DEBUG + STATE + CONSTANTS
-# ═══════════════════════════════════════════════
-DEBUG = True
-debug = {True: print, False: lambda *a, **k: None}[DEBUG]
-
-CHECKPOINT_DIR = "/tmp/checkpoints"
-os.makedirs(CHECKPOINT_DIR, exist_ok=True)
-
-NEAREST_K = 5
-SENT_SPLIT_RE = r'(?<=[.!?])\s+(?=[A-Z])'
-MIN_SENT_LEN = 30
-
-RUN_CONFIGS = {
-    "abstract": ["Abstract"],
-    "title": ["Title"],
-}
-
-_data = {}
-
-
-# ═══════════════════════════════════════════════
-# HELPER: Split text into sentences (regex, no nltk)
-# ═══════════════════════════════════════════════
-def _split_sentences(text):
-    """Split text on sentence boundaries. Filters short fragments (<30 chars).
-    Uses regex: split after .!? followed by uppercase letter."""
-    raw = re.split(SENT_SPLIT_RE, str(text))
-    return list(filter(lambda s: len(s.strip()) >= MIN_SENT_LEN, raw))
-
-
-# ═══════════════════════════════════════════════
-# TOOL 1: Load Scopus CSV
-# ═══════════════════════════════════════════════
-@tool
-def load_scopus_csv(filepath: str) -> str:
-    """Load a Scopus CSV export and show preview. Call this first.
-
-    Args:
-        filepath: Path to the uploaded .csv file.
-
-    Returns:
-        Row count, column names, and sample data."""
-    debug(f"\n>>> TOOL: load_scopus_csv(filepath='{filepath}')")
-    df = pd.read_csv(filepath, encoding="utf-8-sig")
-    _data["df"] = df
-    debug(f">>> Loaded {len(df)} rows, {len(df.columns)} columns")
-    target_cols = list(filter(lambda c: c in df.columns, ["Title", "Abstract", "Author Keywords"]))
-    sample = df[target_cols].head(3).to_string(max_colwidth=80)
-    null_counts = ", ".join(list(map(
-        lambda c: f"{c}: {df[c].notna().sum()}/{len(df)}", target_cols)))
-
-    # Estimate sentence counts
-    sample_sents = df["Abstract"].head(5).apply(_split_sentences).apply(len)
-    avg_abstract_sents = sample_sents.mean()
-    est_abstract = int(avg_abstract_sents * len(df))
-    title_count = int(df["Title"].notna().sum())
-
-    return (f"📊 **Dataset Statistics:**\n"
-            f"- **Papers:** {len(df)}\n"
-            f"- **Abstract sentences:** ~{est_abstract} (~{avg_abstract_sents:.0f} per paper)\n"
-            f"- **Title sentences:** {title_count} (1 per paper)\n"
-            f"- **Non-null:** {null_counts}\n\n"
-            f"Columns: {', '.join(list(df.columns)[:15])}\n\n"
-            f"Sample:\n{sample}")
-
-
-# ═══════════════════════════════════════════════
-# TOOL 2: Sentence-Level BERTopic Pipeline
-# ═══════════════════════════════════════════════
-@tool
-def run_bertopic_discovery(run_key: str, threshold: float = 0.7) -> str:
-    """Sentence-level BERTopic: split papers → embed sentences → cosine similarity clustering → centroid nearest 5 → Plotly charts.
-    Each sentence keeps paper_id, sent_id, and metadata. One paper can span multiple topics.
-    Uses AgglomerativeClustering with cosine distance — groups sentences by similarity threshold.
-
-    Args:
-        run_key: One of 'abstract' or 'title' — selects which columns to split into sentences.
-        threshold: Cosine distance threshold (0.0-1.0). Lower = stricter = more topics.
-                   0.5 = very strict (~2000 topics), 0.7 = recommended (~100 topics, default), 0.8 = loose (~30 topics), 0.9 = very loose (~10 topics).
-
-    Returns:
-        Topic summary with sentence counts, paper counts, and 5 nearest centroid sentences."""
-    debug(f"\n>>> TOOL: run_bertopic_discovery(run_key='{run_key}', threshold={threshold})")
-    from bertopic import BERTopic
-    from sentence_transformers import SentenceTransformer
-
-    df = _data["df"].copy()
-    cols = RUN_CONFIGS[run_key]
-    available = list(filter(lambda c: c in df.columns, cols))
-    debug(f">>> Columns: {available}")
-
-    # ── Step 1: Assemble text per paper ──
-    df["_text"] = df[available].fillna("").agg(" ".join, axis=1)
-    df["_paper_id"] = df.index
-    debug(f">>> {len(df)} papers assembled")
-
-    # ── Step 2: Split into sentences — regex, no nltk ──
-    debug(">>> Splitting into sentences...")
-    df["_sentences"] = df["_text"].apply(_split_sentences)
-    debug(f">>> Sentence counts: min={df['_sentences'].apply(len).min()}, "
-          f"max={df['_sentences'].apply(len).max()}, "
-          f"mean={df['_sentences'].apply(len).mean():.1f}")
-
-    # ── Step 3: Explode to sentence-level DataFrame ──
-    meta_cols = ["_paper_id", "Title", "Author Keywords", "_sentences"]
-    available_meta = list(filter(lambda c: c in df.columns, meta_cols))
-    sent_df = df[available_meta].explode("_sentences").rename(
-        columns={"_sentences": "text"}).reset_index(drop=True)
-    sent_df = sent_df.dropna(subset=["text"]).reset_index(drop=True)
-    sent_df["sent_id"] = sent_df.groupby("_paper_id").cumcount()
-
-    # ── Step 3b: Filter out publisher boilerplate sentences ──
-    # Scopus abstracts contain copyright/license noise that clustering picks up as topics.
-    # These are NOT research content — remove before embedding.
-    debug(">>> Filtering publisher boilerplate...")
-    _n_before = len(sent_df)
-    boilerplate_patterns = "|".join([
-        r"Licensee MDPI",
-        r"Published by Informa",
-        r"Published by Elsevier",
-        r"Taylor & Francis",
-        r"Copyright ©",
-        r"Creative Commons",
-        r"open access article",
-        r"Inderscience Enterprises",
-        r"All rights reserved",
-        r"This is an open access",
-        r"distributed under the terms",
-        r"The Author\(s\)",
-        r"Springer Nature",
-        r"Emerald Publishing",
-        r"limitations and future",
-        r"limitations and implications",
-        r"limitations are discussed",
-        r"limitations have been discussed",
-        r"implications are discussed",
-        r"implications were discussed",
-        r"implications are presented",
-        r"concludes with .* implications",
-    ])
-    clean_mask = ~sent_df["text"].str.contains(boilerplate_patterns, case=False, regex=True, na=False)
-    sent_df = sent_df[clean_mask].reset_index(drop=True)
-    sent_df["sent_id"] = sent_df.groupby("_paper_id").cumcount()
-    debug(f">>> Filtered: {_n_before} → {len(sent_df)} sentences ({_n_before - len(sent_df)} boilerplate removed)")
-    n_sentences = len(sent_df)
-    n_papers = len(df)
-    debug(f">>> {n_sentences} sentences from {n_papers} papers")
-
-    # ── Step 4: Embed sentences (384d, L2-normalized) ──
-    # BERTopic FAQ: "normalize them first to force a cosine-related distance metric"
-    # Math: for L2-normalized vectors, euclidean²(a,b) = 2(1 - cos(a,b)) → same clusters as cosine
-    debug(">>> Embedding sentences with all-MiniLM-L6-v2 (L2-normalized)...")
-    docs = sent_df["text"].tolist()
-    embedder = SentenceTransformer("all-MiniLM-L6-v2")
-    embeddings = embedder.encode(docs, show_progress_bar=False, normalize_embeddings=True)
-    debug(f">>> Embeddings: {embeddings.shape}, normalized: True")
-
-    # Save checkpoint
-    np.save(f"{CHECKPOINT_DIR}/rq4_{run_key}_emb.npy", embeddings)
-
-    # ── Step 5: Agglomerative Clustering with COSINE similarity threshold ──
-    # Groups sentences where cosine_distance < threshold → same cluster
-    # No dimension reduction. No density estimation. Pure similarity grouping.
-    debug(f">>> AgglomerativeClustering cosine threshold={threshold} on 384d embeddings...")
-    from sklearn.preprocessing import FunctionTransformer
-    from sklearn.cluster import AgglomerativeClustering
-    no_umap = FunctionTransformer()
-    cluster_model = AgglomerativeClustering(
-        n_clusters=None,
-        metric="cosine",
-        linkage="average",
-        distance_threshold=threshold,
-    )
-    topic_model = BERTopic(
-        hdbscan_model=cluster_model,
-        umap_model=no_umap,
-    )
-    topics, probs = topic_model.fit_transform(docs, embeddings)
-    n_topics = len(set(topics)) - int(-1 in topics)
-    n_outliers = int(np.sum(np.array(topics) == -1))
-    debug(f">>> {n_topics} topics, {n_outliers} outlier sentences")
-
-    # Store for later tools
-    _data[f"{run_key}_model"] = topic_model
-    _data[f"{run_key}_topics"] = np.array(topics)
-    _data[f"{run_key}_embeddings"] = embeddings
-    _data[f"{run_key}_sent_df"] = sent_df
-
-    # ── Step 6: BERTopic Plotly visualizations (skip charts that need 3+ topics) ──
-    debug(f">>> Generating visualizations ({n_topics} topics)...")
-    # visualize_topics() uses UMAP internally → crashes with < 3 topics
-    (n_topics >= 3) and topic_model.visualize_topics().write_html(
-        f"/tmp/rq4_{run_key}_intertopic.html", include_plotlyjs="cdn")
-    # barchart works with 1+ topics
-    (n_topics >= 1) and topic_model.visualize_barchart(
-        top_n_topics=min(10, max(1, n_topics))).write_html(
-        f"/tmp/rq4_{run_key}_bars.html", include_plotlyjs="cdn")
-    # hierarchy needs 2+ topics
-    (n_topics >= 2) and topic_model.visualize_hierarchy().write_html(
-        f"/tmp/rq4_{run_key}_hierarchy.html", include_plotlyjs="cdn")
-    # heatmap needs 2+ topics
-    (n_topics >= 2) and topic_model.visualize_heatmap().write_html(
-        f"/tmp/rq4_{run_key}_heatmap.html", include_plotlyjs="cdn")
-    debug(f">>> Visualizations saved (skipped charts needing more topics)")
-
-    # ── Step 7: Centroid nearest 5 SENTENCES — COSINE similarity ──
-    topics_arr = np.array(topics)
-    topic_info = topic_model.get_topic_info()
-    valid_rows = list(filter(lambda r: r["Topic"] != -1, topic_info.to_dict("records")))
-
-    def _centroid_nearest(row):
-        """Find 5 sentences nearest to topic centroid via cosine similarity."""
-        mask = topics_arr == row["Topic"]
-        member_idx = np.where(mask)[0]
-        member_embs = embeddings[mask]
-        centroid = member_embs.mean(axis=0)
-        # Cosine distance: 1 - cos_sim. For normalized vectors: cos_sim = dot product
-        norms = np.linalg.norm(member_embs, axis=1) * np.linalg.norm(centroid)
-        cosine_sim = (member_embs @ centroid) / (norms + 1e-10)
-        dists = 1 - cosine_sim
-        nearest = np.argsort(dists)[:NEAREST_K]
-
-        # 5 nearest sentences with paper metadata
-        nearest_evidence = list(map(lambda i: {
-            "sentence": str(sent_df.iloc[member_idx[i]]["text"])[:250],
-            "paper_id": int(sent_df.iloc[member_idx[i]]["_paper_id"]),
-            "title": str(sent_df.iloc[member_idx[i]].get("Title", ""))[:150],
-            "keywords": str(sent_df.iloc[member_idx[i]].get("Author Keywords", ""))[:150],
-        }, nearest))
-
-        # Count unique papers in this topic + collect their titles
-        topic_papers_df = sent_df.iloc[member_idx].drop_duplicates(subset=["_paper_id"])
-        unique_papers = len(topic_papers_df)
-        paper_titles = list(map(
-            lambda idx: str(topic_papers_df.iloc[idx].get("Title", ""))[:200],
-            range(min(50, unique_papers))))  # cap at 50 titles per topic
-
-        return {"topic_id": int(row["Topic"]),
-                "sentence_count": int(row["Count"]),
-                "paper_count": int(unique_papers),
-                "top_words": str(row.get("Name", ""))[:100],
-                "nearest": nearest_evidence,
-                "paper_titles": paper_titles}
-
-    summaries = list(map(_centroid_nearest, valid_rows))
-    json.dump(summaries, open(f"{CHECKPOINT_DIR}/rq4_{run_key}_summaries.json", "w"), indent=2, default=str)
-    debug(f">>> {len(summaries)} topics saved ({NEAREST_K} nearest sentences each)")
-
-    # ── Format output ──
-    lines = list(map(
-        lambda s: f"  Topic {s['topic_id']} ({s['sentence_count']} sentences, {s['paper_count']} papers): {s['top_words']}",
-        summaries))
-    return (f"[{run_key}] {n_topics} topics from {n_sentences} sentences ({n_papers} papers, {n_outliers} outliers).\n\n"
-            + "\n".join(lines)
-            + f"\n\nVisualizations: /tmp/rq4_{run_key}_*.html (4 files)"
-            + f"\nCheckpoints: {CHECKPOINT_DIR}/rq4_{run_key}_emb.npy + summaries.json")
-
-
-# ═══════════════════════════════════════════════
-# TOOL 3: Label Topics with Mistral (sentence evidence)
-# ═══════════════════════════════════════════════
-@tool
-def label_topics_with_llm(run_key: str) -> str:
-    """Send 5 nearest centroid sentences + paper metadata to Mistral for labeling.
-    Each sentence shows which paper it came from (title + keywords).
-
-    Args:
-        run_key: One of 'abstract' or 'title'.
-
-    Returns:
-        Labeled topics with sentence-level evidence."""
-    debug(f"\n>>> TOOL: label_topics_with_llm(run_key='{run_key}')")
-    from langchain_mistralai import ChatMistralAI
-    from langchain_core.prompts import PromptTemplate
-    from langchain_core.output_parsers import JsonOutputParser
-
-    summaries = json.load(open(f"{CHECKPOINT_DIR}/rq4_{run_key}_summaries.json"))
-    debug(f">>> Loaded {len(summaries)} topics ({NEAREST_K} sentences each)")
-
-    # Limit to top 50 largest topics — prevents Mistral rate limit on 2000+ topics
-    MAX_LABEL_TOPICS = 100
-    sorted_summaries = sorted(summaries, key=lambda s: s.get("sentence_count", 0), reverse=True)
-    summaries_to_label = sorted_summaries[:MAX_LABEL_TOPICS]
-    skipped = max(0, len(summaries) - MAX_LABEL_TOPICS)
-    debug(f">>> Labeling top {len(summaries_to_label)} topics (skipped {skipped} small clusters)")
-
-    # Format all topics — show sentence + paper metadata as evidence
-    topics_block = "\n\n".join(list(map(
-        lambda s: (f"Topic {s['topic_id']} ({s['sentence_count']} sentences from {s['paper_count']} papers):\n"
-                   f"  Top words: {s['top_words']}\n"
-                   f"  {NEAREST_K} nearest centroid sentences:\n"
-                   + "\n".join(list(map(
-                       lambda e: (f"    - \"{e['sentence'][:200]}\"\n"
-                                  f"      Paper: \"{e['title']}\"\n"
-                                  f"      Keywords: {e['keywords']}"),
-                       s["nearest"])))),
-        summaries_to_label)))
-
-    prompt = PromptTemplate.from_template(
-        "You are a research topic classifier for academic papers about Technology and Tourism.\n\n"
-        "For EACH topic below, you are given the 5 sentences nearest to the topic centroid,\n"
-        "plus the paper title and author keywords each sentence came from.\n\n"
-        "Return a JSON ARRAY with one object per topic:\n"
-        "- topic_id: integer\n"
-        "- label: short descriptive name (3-6 words, specific — NOT generic like 'tourism studies')\n"
-        "- category: general research area (e.g., 'technology adoption', 'consumer behavior',\n"
-        "    'virtual reality', 'social media marketing', 'sustainability', 'cultural heritage',\n"
-        "    'AI and machine learning', 'online reviews', 'destination marketing',\n"
-        "    'tourist psychology', 'hotel management', 'sharing economy',\n"
-        "    'mobile applications', 'research methodology', 'data analytics')\n"
-        "    DO NOT use PACIS/ICIS categories — just plain descriptive research area.\n"
-        "- confidence: high, medium, or low\n"
-        "- reasoning: 1 sentence explaining WHY you chose this label based on the evidence sentences\n"
-        "- niche: true or false (true = very specific sub-area with <20 sentences)\n\n"
-        "CRITICAL: be SPECIFIC in labels. Do NOT use broad terms.\n"
-        "Return ONLY valid JSON array, no markdown.\n\n"
-        "Topics:\n{topics}")
-
-    llm = ChatMistralAI(model="mistral-small-latest", temperature=0, timeout=300)
-    chain = prompt | llm | JsonOutputParser()
-    debug(">>> Calling Mistral (single call, all topics)...")
-    labels = chain.invoke({"topics": topics_block})
-    debug(f">>> Got {len(labels)} labels")
-
-    # Merge labels with summaries
-    labeled = list(map(lambda pair: {**pair[0], **pair[1]},
-                       zip(summaries, (labels + summaries)[:len(summaries)])))
-    json.dump(labeled, open(f"{CHECKPOINT_DIR}/rq4_{run_key}_labels.json", "w"), indent=2, default=str)
-    debug(f">>> Labels saved: {CHECKPOINT_DIR}/rq4_{run_key}_labels.json")
-
-    # Format — show label + evidence sentences + paper source
-    lines = list(map(
-        lambda l: (f"  **Topic {l.get('topic_id', '?')}: {l.get('label', '?')}** "
-                   f"[{l.get('category', '?')}] conf={l.get('confidence', '?')} "
-                   f"({l.get('sentence_count', 0)} sentences, {l.get('paper_count', 0)} papers)\n"
-                   + "\n".join(list(map(
-                       lambda e: f"    → \"{e['sentence'][:120]}...\" — _{e['title'][:60]}_",
-                       l.get("nearest", []))))),
-        labeled))
-    return f"[{run_key}] {len(labeled)} topics labeled by Mistral:\n\n" + "\n\n".join(lines)
-
-
-# ═══════════════════════════════════════════════
-# TOOL 4: Generate Comparison Table
-# ═══════════════════════════════════════════════
-@tool
-def generate_comparison_csv() -> str:
-    """Compare Mistral-labeled topics across completed runs. Includes sentence + paper counts.
-
-    Returns:
-        Comparison table + CSV path."""
-    debug(f"\n>>> TOOL: generate_comparison_csv()")
-    completed = list(filter(
-        lambda k: os.path.exists(f"{CHECKPOINT_DIR}/rq4_{k}_labels.json"), RUN_CONFIGS.keys()))
-    debug(f">>> Completed runs: {completed}")
-
-    def _load_run(run_key):
-        labels = json.load(open(f"{CHECKPOINT_DIR}/rq4_{run_key}_labels.json"))
-        return list(map(lambda l: {
-            "run": run_key, "topic_id": l.get("topic_id", ""),
-            "label": l.get("label", ""), "category": l.get("category", ""),
-            "confidence": l.get("confidence", ""), "niche": l.get("niche", ""),
-            "sentences": l.get("sentence_count", 0),
-            "papers": l.get("paper_count", 0),
-            "top_words": l.get("top_words", ""),
-        }, labels))
-
-    all_rows = sum(list(map(_load_run, completed)), [])
-    df = pd.DataFrame(all_rows)
-    path = "/tmp/rq4_comparison.csv"
-    df.to_csv(path, index=False)
-    debug(f">>> Comparison CSV: {path} ({len(df)} rows)")
-    return f"Comparison saved: {path} ({len(completed)} runs, {len(df)} topics)\n\n{df.to_string(index=False)}"
-
-
-# ═══════════════════════════════════════════════
-# TOOL 5: Export 500-Word Narrative
-# ═══════════════════════════════════════════════
-@tool
-def export_narrative(run_key: str) -> str:
-    """Generate 500-word narrative for research paper Section 7 via Mistral.
-
-    Args:
-        run_key: One of 'abstract' or 'title'.
-
-    Returns:
-        500-word narrative + save path."""
-    debug(f"\n>>> TOOL: export_narrative(run_key='{run_key}')")
-    from langchain_mistralai import ChatMistralAI
-
-    labels = json.load(open(f"{CHECKPOINT_DIR}/rq4_{run_key}_labels.json"))
-    topics_text = "\n".join(list(map(
-        lambda l: f"- {l.get('label', '?')} ({l.get('sentence_count', 0)} sentences from "
-                  f"{l.get('paper_count', 0)} papers, category: {l.get('category', '?')}, "
-                  f"confidence: {l.get('confidence', '?')}, niche: {l.get('niche', '?')})",
-        labels)))
-
-    llm = ChatMistralAI(model="mistral-small-latest", temperature=0.3, timeout=300)
-    result = llm.invoke(
-        f"Write exactly 500 words for a research paper Section 7 titled "
-        f"'Topic Modeling Results — BERTopic Discovery'.\n\n"
-        f"Dataset: 1390 Scopus papers on Tourism and AI.\n"
-        f"Method: Sentence-level BERTopic — each abstract split into sentences,\n"
-        f"embedded with all-MiniLM-L6-v2 (384d), clustered with AgglomerativeClustering (cosine).\n"
-        f"Note: One paper can contribute sentences to MULTIPLE topics.\n"
-        f"Run config: '{run_key}' columns.\n\n"
-        f"Topics discovered:\n{topics_text}\n\n"
-        f"Include: methodology justification for sentence-level approach,\n"
-        f"key themes, emerging niches, limitations, future work.")
-
-    path = "/tmp/rq4_narrative.txt"
-    open(path, "w", encoding="utf-8").write(result.content)
-    debug(f">>> Narrative saved: {path} ({len(result.content)} chars)")
-    return f"Narrative saved: {path}\n\n{result.content}"
-
-
-# ═══════════════════════════════════════════════
-# TOOL 6: Consolidate Round 1 Topics into Themes
-# ═══════════════════════════════════════════════
-@tool
-def consolidate_into_themes(run_key: str, theme_map: dict) -> str:
-    """ROUND 2: Merge fine-grained Round 1 topics into broader themes.
-    Researcher decides which topics to group. Recomputes centroids and evidence.
-
-    Args:
-        run_key: 'abstract' or 'title'.
-        theme_map: Dict mapping theme names to topic ID lists.
-                   Example: {"AI in Tourism": [0, 1, 5], "VR Tourism": [2, 3]}
-
-    Returns:
-        Consolidated themes with new 5-nearest sentence evidence per theme."""
-    debug(f"\n>>> TOOL: consolidate_into_themes(run_key='{run_key}', {len(theme_map)} themes)")
-
-    topics_arr = _data[f"{run_key}_topics"]
-    embeddings = _data[f"{run_key}_embeddings"]
-    sent_df = _data[f"{run_key}_sent_df"]
-
-    def _build_theme(item):
-        """Merge listed topics into one theme. Recompute centroid + 5 nearest."""
-        theme_name, topic_ids = item
-        mask = np.isin(topics_arr, topic_ids)
-        member_idx = np.where(mask)[0]
-        member_embs = embeddings[mask]
-        centroid = member_embs.mean(axis=0)
-        norms = np.linalg.norm(member_embs, axis=1) * np.linalg.norm(centroid)
-        cosine_sim = (member_embs @ centroid) / (norms + 1e-10)
-        dists = 1 - cosine_sim
-        nearest = np.argsort(dists)[:NEAREST_K]
-
-        nearest_evidence = list(map(lambda i: {
-            "sentence": str(sent_df.iloc[member_idx[i]]["text"])[:250],
-            "paper_id": int(sent_df.iloc[member_idx[i]]["_paper_id"]),
-            "title": str(sent_df.iloc[member_idx[i]].get("Title", ""))[:150],
-            "keywords": str(sent_df.iloc[member_idx[i]].get("Author Keywords", ""))[:150],
-        }, nearest))
-
-        unique_papers = sent_df.iloc[member_idx]["_paper_id"].nunique()
-
-        # Collect paper titles (up to 50)
-        topic_papers_df = sent_df.iloc[member_idx].drop_duplicates(subset=["_paper_id"])
-        paper_titles = list(map(
-            lambda idx: str(topic_papers_df.iloc[idx].get("Title", ""))[:200],
-            range(min(50, len(topic_papers_df)))))
-
-        return {"label": theme_name, "merged_topics": list(topic_ids),
-                "sentence_count": int(mask.sum()), "paper_count": int(unique_papers),
-                "nearest": nearest_evidence, "paper_titles": paper_titles}
-
-    # Add topic_id to each theme (sequential)
-    themes_raw = list(map(_build_theme, theme_map.items()))
-    themes = list(map(
-        lambda pair: {**pair[1], "topic_id": pair[0]},
-        enumerate(themes_raw)))
-    json.dump(themes, open(f"{CHECKPOINT_DIR}/rq4_{run_key}_themes.json", "w"), indent=2, default=str)
-    debug(f">>> {len(themes)} themes saved: {CHECKPOINT_DIR}/rq4_{run_key}_themes.json")
-
-    # Format — show theme + merged topics + evidence
-    lines = list(map(
-        lambda t: (f"  **{t['label']}** ({t['sentence_count']} sentences, {t['paper_count']} papers)\n"
-                   f"    Merged from topics: {t['merged_topics']}\n"
-                   f"    Evidence:\n"
-                   + "\n".join(list(map(
-                       lambda e: f"      → \"{e['sentence'][:120]}...\" — _{e['title'][:60]}_",
-                       t["nearest"])))),
-        themes))
-    return f"[{run_key}] Round 2: {len(themes)} themes consolidated:\n\n" + "\n\n".join(lines)
-
-
-# ═══════════════════════════════════════════════
-# TOOL 7: Compare Themes with PAJAIS Taxonomy
-# ═══════════════════════════════════════════════
-
-# Established IS topic taxonomy from:
-# Jiang, Liang & Tsai (2019) "Knowledge Profile in PAJAIS"
-# Pacific Asia Journal of the AIS, 11(1), 1-24. doi:10.17705/1pais.11101
-PAJAIS_TAXONOMY = [
-    "Electronic and Mobile Business / Social Commerce",
-    "Human Behavior and IS / Human-Computer Interaction",
-    "IS/IT Strategy, Leadership, Governance",
-    "Business Intelligence and Data Analytics",
-    "Design Science and IS",
-    "Enterprise Systems and BPM",
-    "IS Implementation, Adoption, and Diffusion",
-    "Social Media and Business Impact",
-    "Cultural and Global Issues in IS",
-    "IS Security and Privacy",
-    "IS Smart / IoT",
-    "Knowledge Management",
-    "ICT / Digital Platform / IT and Work",
-    "IS Healthcare",
-    "IT Project Management",
-    "Service Science and IS",
-    "Social and Organizational Aspects of IS",
-    "Research Methods and Philosophy",
-    "E-Finance / Economics of IS",
-    "E-Government",
-    "IS Education and Learning",
-    "Green IT and Sustainability",
-]
-
-
-@tool
-def compare_with_taxonomy(run_key: str) -> str:
-    """Compare BERTopic themes against established PAJAIS/PACIS taxonomy
-    (Jiang, Liang & Tsai, 2019). Identifies which themes map to known
-    categories and which are NOVEL/EMERGING (not in existing taxonomy).
-    Researcher reviews mapping and approves new theme consolidation.
-
-    Args:
-        run_key: 'abstract' or 'title'.
-
-    Returns:
-        Mapping table: BERTopic theme → PAJAIS category (or NOVEL)."""
-    debug(f"\n>>> TOOL: compare_with_taxonomy(run_key='{run_key}')")
-    from langchain_mistralai import ChatMistralAI
-    from langchain_core.prompts import PromptTemplate
-    from langchain_core.output_parsers import JsonOutputParser
-
-    # Load themes (prefer consolidated themes, fall back to labels)
-    themes_path = f"{CHECKPOINT_DIR}/rq4_{run_key}_themes.json"
-    labels_path = f"{CHECKPOINT_DIR}/rq4_{run_key}_labels.json"
-    source_path = (os.path.exists(themes_path) and themes_path) or labels_path
-    themes = json.load(open(source_path))
-    debug(f">>> Loaded {len(themes)} themes from {source_path}")
-
-    # Format themes for Mistral
-    themes_text = "\n".join(list(map(
-        lambda t: f"- {t.get('label', '?')} "
-                  f"({t.get('paper_count', t.get('count', '?'))} papers)",
-        themes)))
-
-    taxonomy_text = "\n".join(list(map(lambda c: f"- {c}", PAJAIS_TAXONOMY)))
-
-    prompt = PromptTemplate.from_template(
-        "You are an IS research taxonomy expert.\n\n"
-        "Compare each BERTopic theme against the established PAJAIS/PACIS taxonomy.\n"
-        "For EACH theme, return a JSON ARRAY with:\n"
-        "- label: the BERTopic theme name\n"
-        "- pajais_match: closest PAJAIS category (or 'NOVEL' if no match)\n"
-        "- match_confidence: high, medium, low, or none\n"
-        "- reasoning: why this mapping (1 sentence)\n"
-        "- is_novel: true if this theme represents an emerging area not in the taxonomy\n\n"
-        "Return ONLY valid JSON array.\n\n"
-        "BERTopic Themes:\n{themes}\n\n"
-        "PAJAIS Taxonomy (Jiang et al., 2019):\n{taxonomy}")
-
-    llm = ChatMistralAI(model="mistral-small-latest", temperature=0, timeout=300)
-    chain = prompt | llm | JsonOutputParser()
-    debug(">>> Calling Mistral for taxonomy comparison...")
-    mappings = chain.invoke({"themes": themes_text, "taxonomy": taxonomy_text})
-    debug(f">>> Got {len(mappings)} mappings")
-
-    # Save mapping
-    json.dump(mappings, open(f"{CHECKPOINT_DIR}/rq4_{run_key}_taxonomy_map.json", "w"), indent=2, default=str)
-
-    # Count novel vs mapped
-    novel = list(filter(lambda m: m.get("is_novel", False), mappings))
-    mapped = list(filter(lambda m: not m.get("is_novel", False), mappings))
-
-    # Format output
-    mapped_lines = list(map(
-        lambda m: f"  ✅ {m.get('label', '?')} → **{m.get('pajais_match', '?')}** "
-                  f"(conf={m.get('match_confidence', '?')}) _{m.get('reasoning', '')}_",
-        mapped))
-    novel_lines = list(map(
-        lambda m: f"  🆕 **{m.get('label', '?')}** → NOVEL "
-                  f"_{m.get('reasoning', '')}_",
-        novel))
-
-    return (f"[{run_key}] Taxonomy comparison (Jiang et al., 2019):\n\n"
-            f"**Mapped to PAJAIS categories ({len(mapped)}):**\n" + "\n".join(mapped_lines) +
-            f"\n\n**NOVEL / Emerging themes ({len(novel)}):**\n" + "\n".join(novel_lines) +
-            f"\n\nSaved: {CHECKPOINT_DIR}/rq4_{run_key}_taxonomy_map.json")
-
-
-# ═══════════════════════════════════════════════
-# GET ALL TOOLS
-# ═══════════════════════════════════════════════
-def get_all_tools():
-    """Return all 7 tools with error handling enabled."""
-    tools = [load_scopus_csv, run_bertopic_discovery, label_topics_with_llm,
-             consolidate_into_themes, compare_with_taxonomy,
-             generate_comparison_csv, export_narrative]
-    list(map(lambda t: setattr(t, 'handle_tool_error', True), tools))
-    debug(f">>> tools.py: {len(tools)} tools ready (handle_tool_error=True)")
-    list(map(lambda t: debug(f">>>   - {t.name}"), tools))
-    return tools