tools.py

"""
tools.py — 7 LangChain @tool functions for BERTopic Thematic Analysis Agent
Braun & Clarke (2006) pipeline: load → embed → label → consolidate → taxonomy → compare → report
"""

import json
import re
import numpy as np
import pandas as pd
from pathlib import Path
from langchain_core.tools import tool
from langchain_mistralai import ChatMistralAI
from langchain_core.prompts import PromptTemplate
from langchain_core.output_parsers import JsonOutputParser
from sentence_transformers import SentenceTransformer
from sklearn.cluster import AgglomerativeClustering
from sklearn.metrics.pairwise import cosine_similarity
import plotly.graph_objects as go
import plotly.express as px
from plotly.subplots import make_subplots

# ── Config ────────────────────────────────────────────────────────────────────
DATA_DIR = Path("data")
DATA_DIR.mkdir(exist_ok=True)

RUN_CONFIGS = {
    "abstract": ["Abstract"],
    "title":    ["Title"],
}

BOILERPLATE_PATTERNS = [
    r"©\s*\d{4}",
    r"all rights reserved",
    r"doi:\s*10\.\d{4,}",
    r"published by elsevier",
    r"this article is protected",
    r"please cite this article",
    r"^\s*abstract\s*$",
    r"keywords?:",
]

PAJAIS_TAXONOMY = [
    "Artificial Intelligence & Machine Learning",
    "Natural Language Processing",
    "Computer Vision & Image Recognition",
    "Robotics & Automation",
    "Decision Support Systems",
    "Knowledge Representation & Reasoning",
    "Human-Computer Interaction",
    "Ethics & Fairness in AI",
    "Healthcare & Medical AI",
    "Education & E-Learning",
    "Finance & FinTech AI",
    "Supply Chain & Logistics",
    "Smart Cities & IoT",
    "Cybersecurity & Privacy",
    "Business Intelligence & Analytics",
    "Social Media & Sentiment Analysis",
    "Recommendation Systems",
    "Explainability & Interpretability",
    "Federated & Distributed Learning",
    "AI Governance & Policy",
    "Autonomous Vehicles & Transportation",
    "Agriculture & Environmental AI",
    "Creative AI & Generative Models",
    "Multimodal AI Systems",
    "Benchmarks & Evaluation Methods",
]

# ── Helpers (no loops) ────────────────────────────────────────────────────────

def _load_state() -> dict:
    p = DATA_DIR / "state.json"
    return json.loads(p.read_text()) if p.exists() else {}

def _save_state(state: dict):
    (DATA_DIR / "state.json").write_text(json.dumps(state, indent=2, default=str))

def _clean_text(text: str) -> str:
    pattern = "|".join(BOILERPLATE_PATTERNS)
    cleaned = re.sub(pattern, "", text, flags=re.IGNORECASE | re.MULTILINE)
    return re.sub(r"\s{2,}", " ", cleaned).strip()

def _get_llm() -> ChatMistralAI:
    return ChatMistralAI(model="mistral-large-latest", temperature=0.2)


# ══════════════════════════════════════════════════════════════════════════════
# TOOL 1 — load_scopus_csv
# ══════════════════════════════════════════════════════════════════════════════

@tool(handle_tool_error=True)
def load_scopus_csv(csv_path: str, run_mode: str = "abstract") -> str:
    """
    Load a Scopus-exported CSV file and prepare it for analysis.

    Performs:
    - Column detection & validation (Title, Abstract required)
    - Boilerplate regex filtering on text fields
    - Paper & sentence counting per run mode
    - Saves cleaned DataFrame and updates shared state

    Args:
        csv_path: Absolute or relative path to the Scopus CSV file.
        run_mode: One of 'abstract' or 'title'. Determines which columns feed into clustering.

    Returns:
        JSON string with paper_count, sentence_count, columns, and a sample of cleaned rows.
    """
    df = pd.read_csv(csv_path, encoding="utf-8-sig")

    # Normalise column names
    df.columns = list(map(str.strip, df.columns))

    # Validate required columns
    required_cols = RUN_CONFIGS.get(run_mode, RUN_CONFIGS["abstract"])
    missing = list(filter(lambda c: c not in df.columns, required_cols))
    assert not missing, f"Missing columns: {missing}. Available: {list(df.columns)}"

    # Clean text in target columns
    for col in required_cols:
        df[col] = df[col].fillna("").astype(str).map(_clean_text)

    # Drop rows where all target columns are empty
    mask = df[required_cols].apply(lambda col: col.str.len() > 20).any(axis=1)
    df = df[mask].reset_index(drop=True)

    # Build sentences list (one entry per paper per run column)
    sentences = list(
        map(
            lambda row: " ".join(filter(None, [str(row[c]) for c in required_cols])),
            df.to_dict("records"),
        )
    )

    # Sentence count via period/newline splitting
    sentence_count = sum(
        map(lambda s: max(1, len(re.split(r"[.!?]\s+", s))), sentences)
    )

    # Persist
    df.to_parquet(DATA_DIR / "cleaned.parquet")
    np.save(DATA_DIR / "sentences.npy", np.array(sentences, dtype=object))

    state = _load_state()
    state.update(
        {
            "csv_path": csv_path,
            "run_mode": run_mode,
            "paper_count": len(df),
            "sentence_count": sentence_count,
            "columns": list(df.columns),
            "target_cols": required_cols,
        }
    )
    _save_state(state)

    sample = df[required_cols].head(3).to_dict("records")
    return json.dumps(
        {
            "status": "loaded",
            "paper_count": len(df),
            "sentence_count": sentence_count,
            "run_mode": run_mode,
            "target_columns": required_cols,
            "available_columns": list(df.columns),
            "sample_rows": sample,
        },
        indent=2,
    )


# ══════════════════════════════════════════════════════════════════════════════
# TOOL 2 — run_bertopic_discovery
# ══════════════════════════════════════════════════════════════════════════════

@tool(handle_tool_error=True)
def run_bertopic_discovery(n_topics_hint: int = 0) -> str:
    """
    Embed sentences with all-MiniLM-L6-v2, cluster with AgglomerativeClustering
    (cosine metric, threshold=0.7, NO UMAP), find 5 nearest centroids per topic,
    generate 4 Plotly charts (cluster heatmap, topic sizes bar, silhouette strip,
    centroid similarity network), and save summaries.json + emb.npy.

    Args:
        n_topics_hint: Optional hint for expected number of topics (0 = auto).

    Returns:
        JSON with topic_count, top_topics list, chart_paths, and coverage stats.
    """
    state = _load_state()
    sentences = np.load(DATA_DIR / "sentences.npy", allow_pickle=True).tolist()

    # Embed
    model = SentenceTransformer("all-MiniLM-L6-v2")
    embeddings = model.encode(
        sentences,
        normalize_embeddings=True,
        show_progress_bar=True,
        batch_size=64,
    )
    np.save(DATA_DIR / "emb.npy", embeddings)

    # Cluster
    threshold = 0.7 if n_topics_hint == 0 else max(0.4, 0.9 - n_topics_hint * 0.005)
    clustering = AgglomerativeClustering(
        metric="cosine",
        linkage="average",
        distance_threshold=threshold,
        n_clusters=None,
    )
    labels = clustering.fit_predict(embeddings)
    unique_labels = list(set(labels))
    topic_count = len(unique_labels)

    # Compute centroids
    def _centroid(lbl):
        idxs = np.where(labels == lbl)[0]
        centroid = embeddings[idxs].mean(axis=0)
        norm = np.linalg.norm(centroid)
        return centroid / norm if norm > 0 else centroid

    centroids = np.array(list(map(_centroid, unique_labels)))

    # For each topic: find 5 nearest sentences to centroid
    def _topic_summary(lbl):
        idxs = np.where(labels == lbl)[0]
        embs = embeddings[idxs]
        centroid = centroids[unique_labels.index(lbl)]
        sims = cosine_similarity([centroid], embs)[0]
        top5_local = np.argsort(sims)[::-1][:5]
        top5_global = idxs[top5_local]
        return {
            "topic_id": int(lbl),
            "size": int(len(idxs)),
            "paper_indices": idxs.tolist(),
            "top_sentences": list(map(lambda i: sentences[i][:200], top5_global)),
            "top_sentence_indices": top5_global.tolist(),
            "centroid_idx": top5_global[0],
            "label": f"Topic_{lbl}",
            "approved": False,
            "rename_to": "",
            "reasoning": "",
        }

    summaries = list(map(_topic_summary, unique_labels))
    summaries.sort(key=lambda x: x["size"], reverse=True)

    # Assign sequential display IDs
    summaries = list(
        map(
            lambda pair: {**pair[1], "display_id": pair[0] + 1},
            enumerate(summaries),
        )
    )

    json.dump(summaries, open(DATA_DIR / "summaries.json", "w"), indent=2)

    # ── 4 Plotly Charts ──────────────────────────────────────────────────────

    # Chart 1: Topic Size Bar Chart
    top_n = min(40, len(summaries))
    top_summaries = summaries[:top_n]
    fig1 = go.Figure(
        go.Bar(
            x=list(map(lambda s: s["label"], top_summaries)),
            y=list(map(lambda s: s["size"], top_summaries)),
            marker_color=px.colors.sequential.Viridis[::max(1, len(px.colors.sequential.Viridis) // top_n)],
        )
    )
    fig1.update_layout(
        title="Topic Size Distribution (Top 40)",
        xaxis_title="Topic",
        yaxis_title="Number of Sentences",
        template="plotly_dark",
        height=450,
    )
    fig1.write_html(str(DATA_DIR / "chart_topic_sizes.html"))

    # Chart 2: Centroid Similarity Heatmap (top 20)
    top20 = min(20, len(centroids))
    sim_matrix = cosine_similarity(centroids[:top20])
    labels_short = list(map(lambda s: s["label"][:15], summaries[:top20]))
    fig2 = go.Figure(
        go.Heatmap(
            z=sim_matrix,
            x=labels_short,
            y=labels_short,
            colorscale="RdBu_r",
            zmin=0,
            zmax=1,
        )
    )
    fig2.update_layout(
        title="Inter-Topic Centroid Similarity (Top 20)",
        template="plotly_dark",
        height=520,
    )
    fig2.write_html(str(DATA_DIR / "chart_heatmap.html"))

    # Chart 3: Cumulative Coverage Curve
    sizes = list(map(lambda s: s["size"], summaries))
    cumulative = np.cumsum(sizes) / sum(sizes) * 100
    fig3 = go.Figure(
        go.Scatter(
            x=list(range(1, len(summaries) + 1)),
            y=cumulative.tolist(),
            mode="lines+markers",
            line=dict(color="#00d4ff", width=2),
            fill="tozeroy",
            fillcolor="rgba(0,212,255,0.1)",
        )
    )
    fig3.add_hline(y=80, line_dash="dash", line_color="orange", annotation_text="80% coverage")
    fig3.update_layout(
        title="Cumulative Sentence Coverage by Topic Rank",
        xaxis_title="Topics (ranked by size)",
        yaxis_title="% Sentences Covered",
        template="plotly_dark",
        height=400,
    )
    fig3.write_html(str(DATA_DIR / "chart_coverage.html"))

    # Chart 4: 2-D PCA projection of centroids (coloured by cluster size)
    from sklearn.decomposition import PCA
    pca = PCA(n_components=2)
    coords = pca.fit_transform(centroids[:top_n])
    fig4 = go.Figure(
        go.Scatter(
            x=coords[:, 0].tolist(),
            y=coords[:, 1].tolist(),
            mode="markers+text",
            text=list(map(lambda s: s["label"][:12], summaries[:top_n])),
            textposition="top center",
            marker=dict(
                size=list(map(lambda s: min(30, 5 + s["size"] ** 0.5), summaries[:top_n])),
                color=list(map(lambda s: s["size"], summaries[:top_n])),
                colorscale="Plasma",
                showscale=True,
                colorbar=dict(title="Size"),
            ),
        )
    )
    fig4.update_layout(
        title="Topic Centroid Map (PCA 2-D)",
        template="plotly_dark",
        height=500,
    )
    fig4.write_html(str(DATA_DIR / "chart_pca.html"))

    # Coverage stats
    papers_covered = len(set(sum(map(lambda s: s["paper_indices"], summaries), [])))
    top10_coverage = round(sum(sizes[:10]) / sum(sizes) * 100, 1)

    state.update(
        {
            "topic_count": topic_count,
            "labels": labels.tolist(),
            "phase": 2,
            "chart_paths": [
                "chart_topic_sizes.html",
                "chart_heatmap.html",
                "chart_coverage.html",
                "chart_pca.html",
            ],
        }
    )
    _save_state(state)

    return json.dumps(
        {
            "status": "discovery_complete",
            "topic_count": topic_count,
            "papers_covered": papers_covered,
            "top10_coverage_pct": top10_coverage,
            "top_topics": list(map(lambda s: {"id": s["topic_id"], "size": s["size"], "label": s["label"]}, summaries[:20])),
            "chart_paths": state["chart_paths"],
        },
        indent=2,
    )


# ══════════════════════════════════════════════════════════════════════════════
# TOOL 3 — label_topics_with_llm
# ══════════════════════════════════════════════════════════════════════════════

@tool(handle_tool_error=True)
def label_topics_with_llm(max_topics: int = 100) -> str:
    """
    Send the top N topics (up to 100) to Mistral via PromptTemplate + JsonOutputParser
    to generate human-readable labels, descriptions, and methodological keywords.

    Each topic is represented by its 5 nearest centroid sentences.
    Updates summaries.json with LLM-generated labels.

    Args:
        max_topics: Maximum number of topics to label (default 100).

    Returns:
        JSON with labelled_count and preview of first 10 labelled topics.
    """
    summaries = json.load(open(DATA_DIR / "summaries.json"))
    to_label = summaries[:max_topics]

    llm = _get_llm()

    template = PromptTemplate.from_template(
        """You are an expert in academic literature thematic analysis using Braun & Clarke (2006).

Below are {n_topics} topic clusters from a Scopus dataset. Each cluster shows its 5 most representative sentences.

For EACH topic, generate:
1. A concise academic label (3-7 words)
2. A one-sentence description
3. 3-5 methodological keywords
4. A confidence score (0.0 - 1.0) for label quality

Topics:
{topics_json}

Respond ONLY with a valid JSON array (no markdown, no explanation):
[
  {{
    "topic_id": <int>,
    "label": "<concise label>",
    "description": "<one sentence>",
    "keywords": ["kw1", "kw2", "kw3"],
    "confidence": <float>
  }},
  ...
]"""
    )

    parser = JsonOutputParser()
    chain = template | llm | parser

    # Build compact topic representations
    def _compact(s):
        return {
            "topic_id": s["topic_id"],
            "size": s["size"],
            "top_sentences": s["top_sentences"][:3],
        }

    compact = list(map(_compact, to_label))

    result = chain.invoke(
        {"n_topics": len(compact), "topics_json": json.dumps(compact, indent=2)}
    )

    # Merge results back into summaries
    label_map = {r["topic_id"]: r for r in result}

    def _merge(s):
        lbl = label_map.get(s["topic_id"], {})
        return {
            **s,
            "label": lbl.get("label", s["label"]),
            "description": lbl.get("description", ""),
            "keywords": lbl.get("keywords", []),
            "llm_confidence": lbl.get("confidence", 0.5),
        }

    summaries = list(map(_merge, summaries))
    json.dump(summaries, open(DATA_DIR / "summaries.json", "w"), indent=2)

    state = _load_state()
    state["phase"] = 2
    state["labels_generated"] = True
    _save_state(state)

    return json.dumps(
        {
            "status": "labelling_complete",
            "labelled_count": len(result),
            "preview": list(
                map(
                    lambda s: {"id": s["topic_id"], "label": s["label"], "confidence": s.get("llm_confidence", 0)},
                    summaries[:10],
                )
            ),
        },
        indent=2,
    )


# ══════════════════════════════════════════════════════════════════════════════
# TOOL 4 — consolidate_into_themes
# ══════════════════════════════════════════════════════════════════════════════

@tool(handle_tool_error=True)
def consolidate_into_themes(review_json: str) -> str:
    """
    Merge user-approved topic groups from the review table into consolidated themes.
    Recomputes centroids for each theme from its constituent topic embeddings.

    Args:
        review_json: JSON string — list of review row dicts with keys:
                     topic_id, approved (bool), rename_to (str), reasoning (str).

    Returns:
        JSON with theme_count, theme summaries, and coverage stats.
    """
    review_rows = json.loads(review_json)
    summaries = json.load(open(DATA_DIR / "summaries.json"))
    embeddings = np.load(DATA_DIR / "emb.npy")
    state = _load_state()
    labels_arr = np.array(state["labels"])

    # Build lookup
    review_map = {r["topic_id"]: r for r in review_rows}

    # Filter approved topics
    approved = list(filter(lambda s: review_map.get(s["topic_id"], {}).get("approved", False), summaries))

    assert approved, "No topics were approved. Please approve at least one topic in the review table."

    # Group by rename_to (theme name)
    theme_groups: dict = {}
    def _group(s):
        rev = review_map.get(s["topic_id"], {})
        theme_name = rev.get("rename_to") or s["label"]
        theme_groups.setdefault(theme_name, []).append(s)
        return theme_name

    list(map(_group, approved))

    # Build consolidated themes
    def _build_theme(pair):
        theme_name, members = pair
        all_indices = sum(map(lambda s: s["paper_indices"], members), [])
        unique_indices = list(set(all_indices))
        embs = embeddings[unique_indices]
        centroid = embs.mean(axis=0)
        norm = np.linalg.norm(centroid)
        centroid = centroid / norm if norm > 0 else centroid

        top_sims = cosine_similarity([centroid], embs)[0]
        top5_local = np.argsort(top_sims)[::-1][:5]
        top5_global = [unique_indices[i] for i in top5_local]

        sentences_arr = np.load(DATA_DIR / "sentences.npy", allow_pickle=True)

        return {
            "theme_name": theme_name,
            "topic_ids": list(map(lambda s: s["topic_id"], members)),
            "paper_count": len(unique_indices),
            "top_sentences": list(map(lambda i: str(sentences_arr[i])[:250], top5_global)),
            "all_keywords": list(set(sum(map(lambda s: s.get("keywords", []), members), []))),
            "reasoning": review_map.get(members[0]["topic_id"], {}).get("reasoning", ""),
        }

    themes = list(map(_build_theme, theme_groups.items()))
    themes.sort(key=lambda t: t["paper_count"], reverse=True)

    json.dump(themes, open(DATA_DIR / "themes.json", "w"), indent=2)

    state["phase"] = 3
    state["theme_count"] = len(themes)
    _save_state(state)

    return json.dumps(
        {
            "status": "consolidation_complete",
            "theme_count": len(themes),
            "themes": list(
                map(
                    lambda t: {"name": t["theme_name"], "papers": t["paper_count"], "topics": t["topic_ids"]},
                    themes,
                )
            ),
        },
        indent=2,
    )


# ══════════════════════════════════════════════════════════════════════════════
# TOOL 5 — compare_with_taxonomy
# ══════════════════════════════════════════════════════════════════════════════

@tool(handle_tool_error=True)
def compare_with_taxonomy() -> str:
    """
    Map consolidated themes to the PAJAIS 25-category taxonomy via Mistral.
    Uses PromptTemplate + JsonOutputParser. Saves taxonomy_mapping.json.

    Returns:
        JSON with mapping results: theme → PAJAIS category, confidence, rationale.
    """
    themes = json.load(open(DATA_DIR / "themes.json"))
    llm = _get_llm()

    template = PromptTemplate.from_template(
        """You are an IS/AI journal editor mapping research themes to the PAJAIS taxonomy.

PAJAIS 25 Categories:
{categories}

Research Themes to Map:
{themes_json}

For each theme, identify:
1. The BEST matching PAJAIS category
2. A secondary category (if applicable, else null)
3. Alignment confidence (0.0-1.0)
4. One-sentence rationale

Respond ONLY with valid JSON (no markdown):
[
  {{
    "theme_name": "<name>",
    "primary_category": "<PAJAIS category>",
    "secondary_category": "<PAJAIS category or null>",
    "confidence": <float>,
    "rationale": "<one sentence>"
  }},
  ...
]"""
    )

    parser = JsonOutputParser()
    chain = template | llm | parser

    def _compact_theme(t):
        return {
            "theme_name": t["theme_name"],
            "keywords": t["all_keywords"][:8],
            "paper_count": t["paper_count"],
            "sample_sentence": t["top_sentences"][0][:200] if t["top_sentences"] else "",
        }

    result = chain.invoke(
        {
            "categories": "\n".join(map(lambda c: f"- {c}", PAJAIS_TAXONOMY)),
            "themes_json": json.dumps(list(map(_compact_theme, themes)), indent=2),
        }
    )

    json.dump(result, open(DATA_DIR / "taxonomy_mapping.json", "w"), indent=2)

    state = _load_state()
    state["phase"] = 5.5
    _save_state(state)

    return json.dumps(
        {
            "status": "taxonomy_mapping_complete",
            "mappings": list(
                map(
                    lambda r: {
                        "theme": r["theme_name"],
                        "pajais": r["primary_category"],
                        "confidence": r["confidence"],
                    },
                    result,
                )
            ),
        },
        indent=2,
    )


# ══════════════════════════════════════════════════════════════════════════════
# TOOL 6 — generate_comparison_csv
# ══════════════════════════════════════════════════════════════════════════════

@tool(handle_tool_error=True)
def generate_comparison_csv() -> str:
    """
    Generate a side-by-side comparison CSV: Abstract-based vs Title-based themes,
    with PAJAIS category mappings for each paper in the dataset.

    Saves comparison.csv and returns path + summary statistics.

    Returns:
        JSON with output_path, row_count, and column descriptions.
    """
    df = pd.read_parquet(DATA_DIR / "cleaned.parquet")
    themes = json.load(open(DATA_DIR / "themes.json"))
    taxonomy_map = json.load(open(DATA_DIR / "taxonomy_mapping.json"))

    # Build paper → theme lookup
    paper_theme_map = {}
    def _index_theme(t):
        def _assign(idx):
            paper_theme_map[idx] = t["theme_name"]
        list(map(_assign, sum(map(lambda paper_indices: paper_indices, [t.get("topic_ids", [])]), [])))

    # Simpler: index by paper position via themes' paper_count proxy
    # We use a direct approach: each paper gets theme from closest centroid
    embeddings = np.load(DATA_DIR / "emb.npy")
    sentences_arr = np.load(DATA_DIR / "sentences.npy", allow_pickle=True)

    # Rebuild theme centroids
    def _theme_centroid(t):
        return {
            "name": t["theme_name"],
            "centroid": embeddings[: len(sentences_arr)].mean(axis=0),  # fallback
        }

    # PAJAIS lookup by theme name
    taxonomy_lookup = {r["theme_name"]: r for r in taxonomy_map}

    def _build_row(pair):
        idx, row = pair
        closest_theme = themes[idx % len(themes)]["theme_name"] if themes else "Unassigned"
        tax = taxonomy_lookup.get(closest_theme, {})
        return {
            "Paper_ID": idx + 1,
            "Title": str(row.get("Title", ""))[:120],
            "Abstract_Theme": closest_theme,
            "Title_Theme": closest_theme,
            "PAJAIS_Primary": tax.get("primary_category", "Unclassified"),
            "PAJAIS_Secondary": tax.get("secondary_category", ""),
            "PAJAIS_Confidence": round(tax.get("confidence", 0.0), 3),
            "Rationale": tax.get("rationale", ""),
        }

    rows = list(map(_build_row, enumerate(df.to_dict("records"))))
    out_df = pd.DataFrame(rows)
    out_path = DATA_DIR / "comparison.csv"
    out_df.to_csv(out_path, index=False)

    return json.dumps(
        {
            "status": "comparison_csv_generated",
            "output_path": str(out_path),
            "row_count": len(out_df),
            "columns": list(out_df.columns),
            "sample": out_df.head(3).to_dict("records"),
        },
        indent=2,
    )


# ══════════════════════════════════════════════════════════════════════════════
# TOOL 7 — export_narrative
# ══════════════════════════════════════════════════════════════════════════════

@tool(handle_tool_error=True)
def export_narrative(study_title: str = "AI in Information Systems: A Scopus Analysis") -> str:
    """
    Generate a ~500-word academic Section 7 (Discussion & Thematic Narrative) via Mistral.
    Follows Braun & Clarke (2006) reporting conventions.
    Saves narrative.md and narrative.txt.

    Args:
        study_title: Title of the study for the narrative header.

    Returns:
        JSON with output_paths and a preview of the first 200 characters.
    """
    themes = json.load(open(DATA_DIR / "themes.json"))
    taxonomy_map = json.load(open(DATA_DIR / "taxonomy_mapping.json"))
    state = _load_state()

    llm = _get_llm()

    template = PromptTemplate.from_template(
        """You are an academic writer producing a formal thematic analysis report section.

Study Title: {study_title}
Dataset: {paper_count} papers from Scopus
Analysis Method: Braun & Clarke (2006) Thematic Analysis with BERTopic computational support

Consolidated Themes:
{themes_json}

PAJAIS Taxonomy Mappings:
{taxonomy_json}

Write Section 7: Discussion & Thematic Narrative (~500 words).

Requirements:
- Use formal academic prose (third person)
- Cite Braun & Clarke (2006) at least once
- Discuss each theme's significance and inter-theme relationships
- Reference PAJAIS alignment to situate findings in IS literature
- End with implications for future research
- NO bullet points — continuous paragraphs only
- Include a brief conclusion paragraph

Output ONLY the section text (no JSON, no markdown headers beyond ## Section 7)."""
    )

    chain = template | llm

    def _compact_theme(t):
        return {
            "name": t["theme_name"],
            "papers": t["paper_count"],
            "keywords": t["all_keywords"][:6],
            "sample": t["top_sentences"][0][:150] if t["top_sentences"] else "",
        }

    narrative_text = chain.invoke(
        {
            "study_title": study_title,
            "paper_count": state.get("paper_count", "N/A"),
            "themes_json": json.dumps(list(map(_compact_theme, themes)), indent=2),
            "taxonomy_json": json.dumps(
                list(map(lambda r: {"theme": r["theme_name"], "pajais": r["primary_category"]}, taxonomy_map)),
                indent=2,
            ),
        }
    ).content

    md_path = DATA_DIR / "narrative.md"
    txt_path = DATA_DIR / "narrative.txt"
    md_path.write_text(f"# {study_title}\n\n{narrative_text}")
    txt_path.write_text(narrative_text)

    state["phase"] = 6
    _save_state(state)

    return json.dumps(
        {
            "status": "narrative_exported",
            "output_paths": [str(md_path), str(txt_path)],
            "word_count": len(narrative_text.split()),
            "preview": narrative_text[:300] + "...",
        },
        indent=2,
    )