Upload 4 files

README.md

@@ -1,32 +1,64 @@
----
-title: CHB BERTopic Analysis
-emoji: 🔬
-colorFrom: blue
-colorTo: indigo
-sdk: gradio
-sdk_version: "5.31.0"
-app_file: app.py
-pinned: false
-python_version: "3.11"
----
-
-# CHB Journal BERTopic Analysis
-
-Agentic AI topic modelling for "Computers in Human Behavior" journal using BERTopic and Braun & Clarke thematic analysis.
-
-## Features
-
-- **BERTopic Discovery**: Sentence-level embeddings with clustering
-- **LLM Labeling**: Topic labels via Mistral-7B (HuggingFace)
-- **PAJAIS Mapping**: Taxonomy gap analysis (MAPPED vs NOVEL)
-- **Comparison**: Abstract vs Title theme analysis
-
-## Setup
-
-1. Add `HF_TOKEN` secret in Space Settings
-2. Upload Scopus CSV
-3. Run phases 1-7 sequentially
-
-## Author
-
-Prasad Vijay Gade (2024301006) | MDM Assignment-4 | Prof. Shailaja Jha
+# CHB BERTopic App v2
+
+This is the draft_2 BERTopic rebuild for the CHB project.
+
+## What changed
+
+1. uses the full **7,553-paper** CHB corpus
+2. uses real **BERTopic**
+3. uses **SPECTER2** embeddings
+4. uses **UMAP** for dimensionality reduction
+5. uses **HDBSCAN** for topic discovery
+6. removes the old `Topic 3` / `Topic 26` style fallback from final labels
+7. keeps all new outputs inside `DRAFT_2`
+
+## Default input
+
+`DRAFT_2\sources\corpus_7553\ComputersinHumanBehavior_TopicModelling_Export_7553_for_app.csv`
+
+## Default output
+
+`DRAFT_2\bertopic_outputs_7553`
+
+## Run full pipeline
+
+```powershell
+python app.py --mode pipeline
+```
+
+By default the app now uses the local Ollama model:
+
+`gemma4:e2b`
+
+You can override that with environment variables if needed:
+
+```powershell
+$env:CHB_OLLAMA_MODEL = "gemma4:e2b"
+$env:CHB_OLLAMA_BASE_URL = "http://localhost:11434/v1"
+$env:CHB_OLLAMA_TIMEOUT_SECONDS = "300"
+python app.py --mode pipeline
+```
+
+## Launch UI
+
+```powershell
+python app.py --mode ui
+```
+
+## Output highlights
+
+The pipeline writes:
+
+1. cleaned paper file
+2. embeddings
+3. BERTopic models
+4. topic assignments
+5. human-readable topic labels
+6. consolidated themes
+7. PAJAIS taxonomy mapping
+8. abstract vs title comparison
+9. narrative and audit files
+
+## Important method note
+
+For the abstract-side run, SPECTER2 embeddings are created from **title + abstract** because the model is trained for scientific paper title/abstract inputs. Topic words, representative texts, and labels are still derived from the cleaned abstract text so the output remains content-side rather than title-only.

app.py

@@ -1,351 +1,165 @@
-"""
-app.py - BERTopic Agentic AI Analysis for CHB Journal (HuggingFace Deployment)
-===============================================================================
-Gradio interface for RQ5-7 using HuggingFace Inference API.
-
-Author: Prasad Vijay Gade (2024301006)
-"""
-
-import os
-import gradio as gr
-import pandas as pd
-import json
-from datetime import datetime
-
-from tools import (
-    load_scopus_csv,
-    run_bertopic_discovery,
-    label_topics_with_llm,
-    consolidate_into_themes,
-    compare_with_taxonomy,
-    generate_comparison_csv,
-    export_narrative,
-    OUTPUT_DIR,
-    HF_MODEL
-)
-
-# ============================================================================
-# STATE MANAGEMENT
-# ============================================================================
-
-class AnalysisState:
-    def __init__(self):
-        self.reset()
-    
-    def reset(self):
-        self.phases_completed = {
-            "load": False,
-            "abstract_discovery": False,
-            "abstract_labels": False,
-            "abstract_themes": False,
-            "abstract_taxonomy": False,
-            "title_discovery": False,
-            "title_labels": False,
-            "title_themes": False,
-            "title_taxonomy": False,
-            "comparison": False,
-            "narrative": False
-        }
-        self.logs = []
-    
-    def log(self, message):
-        timestamp = datetime.now().strftime("%H:%M:%S")
-        self.logs.append(f"[{timestamp}] {message}")
-        return "\n".join(self.logs[-20:])
-
-state = AnalysisState()
-
-# ============================================================================
-# PHASE FUNCTIONS
-# ============================================================================
-
-def check_api():
-    token = os.environ.get("HF_TOKEN", "")
-    if not token:
-        return "❌ HF_TOKEN not set. Add in Settings > Secrets."
-    return f"✅ Using {HF_MODEL} via HuggingFace"
-
-def phase_1_load(file):
-    state.log("Phase 1: Loading CSV...")
-    if file is None:
-        return "❌ Upload a CSV file first", state.log("Waiting...")
-    
-    file_path = file.name if hasattr(file, 'name') else str(file)
-    result = load_scopus_csv(file_path)
-    state.phases_completed["load"] = True
-    
-    summary = f"""✅ Phase 1 Complete
-
-📊 Statistics:
-- Papers: {result['total_papers']}
-- Sentences: {result['total_sentences']}
-- Years: {result['year_range']}
-"""
-    return summary, state.log("Ready for Phase 2")
-
-def phase_2_abstract():
-    state.log("Phase 2a: BERTopic on Abstracts...")
-    if not state.phases_completed["load"]:
-        return "❌ Complete Phase 1 first", state.log("Error")
-    
-    result = run_bertopic_discovery("Abstract")
-    state.phases_completed["abstract_discovery"] = True
-    
-    topics_preview = "\n".join([f"  Topic {k}: {v['sample'][:80]}..." 
-                                for k, v in list(result['top_10_topics'].items())[:5]])
-    return f"""✅ Phase 2a Complete
-
-Topics: {result['num_topics']}
-Texts: {result['total_texts']}
-
-Top 5:
-{topics_preview}
-""", state.log("Ready for labeling")
-
-def phase_2_title():
-    state.log("Phase 2b: BERTopic on Titles...")
-    if not state.phases_completed["load"]:
-        return "❌ Complete Phase 1 first", state.log("Error")
-    
-    result = run_bertopic_discovery("Title")
-    state.phases_completed["title_discovery"] = True
-    
-    topics_preview = "\n".join([f"  Topic {k}: {v['sample'][:80]}..." 
-                                for k, v in list(result['top_10_topics'].items())[:5]])
-    return f"""✅ Phase 2b Complete
-
-Topics: {result['num_topics']}
-Texts: {result['total_texts']}
-
-Top 5:
-{topics_preview}
-""", state.log("Ready for labeling")
-
-def phase_3_abstract():
-    state.log("Phase 3a: Labeling abstracts...")
-    if not state.phases_completed["abstract_discovery"]:
-        return "❌ Complete Phase 2a first", state.log("Error")
-    
-    result = label_topics_with_llm("abstract")
-    state.phases_completed["abstract_labels"] = True
-    
-    labels = "\n".join([f"  {k}: {v}" for k, v in list(result['sample_labels'].items())[:10]])
-    return f"""✅ Phase 3a Complete
-
-Labels: {result['num_labeled']}
-
-{labels}
-""", state.log("Ready for consolidation")
-
-def phase_3_title():
-    state.log("Phase 3b: Labeling titles...")
-    if not state.phases_completed["title_discovery"]:
-        return "❌ Complete Phase 2b first", state.log("Error")
-    
-    result = label_topics_with_llm("title")
-    state.phases_completed["title_labels"] = True
-    
-    labels = "\n".join([f"  {k}: {v}" for k, v in list(result['sample_labels'].items())[:10]])
-    return f"""✅ Phase 3b Complete
-
-Labels: {result['num_labeled']}
-
-{labels}
-""", state.log("Ready for consolidation")
-
-def phase_4_abstract():
-    state.log("Phase 4a: Consolidating abstracts...")
-    if not state.phases_completed["abstract_labels"]:
-        return "❌ Complete Phase 3a first", state.log("Error")
-    
-    result = consolidate_into_themes("abstract", 15)
-    state.phases_completed["abstract_themes"] = True
-    
-    themes = "\n".join([f"  {k}: {v} items" for k, v in list(result['themes'].items())[:10]])
-    return f"""✅ Phase 4a Complete
-
-Themes: {result['num_themes']}
-
-{themes}
-""", state.log("Ready for taxonomy")
-
-def phase_4_title():
-    state.log("Phase 4b: Consolidating titles...")
-    if not state.phases_completed["title_labels"]:
-        return "❌ Complete Phase 3b first", state.log("Error")
-    
-    result = consolidate_into_themes("title", 15)
-    state.phases_completed["title_themes"] = True
-    
-    themes = "\n".join([f"  {k}: {v} items" for k, v in list(result['themes'].items())[:10]])
-    return f"""✅ Phase 4b Complete
-
-Themes: {result['num_themes']}
-
-{themes}
-""", state.log("Ready for taxonomy")
-
-def phase_5_abstract():
-    state.log("Phase 5a: PAJAIS mapping abstracts...")
-    if not state.phases_completed["abstract_themes"]:
-        return "❌ Complete Phase 4a first", state.log("Error")
-    
-    result = compare_with_taxonomy("abstract")
-    state.phases_completed["abstract_taxonomy"] = True
-    
-    return f"""✅ Phase 5a Complete
-
-MAPPED: {result['mapped_count']}
-NOVEL: {result['novel_count']}
-
-Novel themes:
-{chr(10).join(['  - ' + t for t in result['novel_themes'][:5]])}
-""", state.log("Complete")
-
-def phase_5_title():
-    state.log("Phase 5b: PAJAIS mapping titles...")
-    if not state.phases_completed["title_themes"]:
-        return "❌ Complete Phase 4b first", state.log("Error")
-    
-    result = compare_with_taxonomy("title")
-    state.phases_completed["title_taxonomy"] = True
-    
-    return f"""✅ Phase 5b Complete
-
-MAPPED: {result['mapped_count']}
-NOVEL: {result['novel_count']}
-
-Novel themes:
-{chr(10).join(['  - ' + t for t in result['novel_themes'][:5]])}
-""", state.log("Ready for comparison")
-
-def phase_6():
-    state.log("Phase 6: Generating comparison...")
-    if not (state.phases_completed["abstract_taxonomy"] and state.phases_completed["title_taxonomy"]):
-        return "❌ Complete Phase 5a and 5b first", state.log("Error")
-    
-    result = generate_comparison_csv()
-    state.phases_completed["comparison"] = True
-    
-    return f"""✅ Phase 6 Complete
-
-Unique Themes: {result['total_unique_themes']}
-In Both: {result['themes_in_both']}
-Abstract-Only: {result['abstract_only']}
-Title-Only: {result['title_only']}
-""", state.log("Ready for narrative")
-
-def phase_7():
-    state.log("Phase 7: Generating narrative...")
-    if not state.phases_completed["comparison"]:
-        return "❌ Complete Phase 6 first", state.log("Error")
-    
-    result = export_narrative()
-    state.phases_completed["narrative"] = True
-    
-    return f"""✅ Phase 7 Complete
-
-Words: {result['word_count']}
-
-{result['preview']}...
-""", state.log("✅ ANALYSIS COMPLETE")
-
-def download_outputs():
-    files = ["comparison.csv", "themes_abstract.json", "themes_title.json",
-             "taxonomy_map_abstract.json", "taxonomy_map_title.json", 
-             "narrative.txt", "comparison_summary.json"]
-    return [f"{OUTPUT_DIR}/{f}" for f in files if os.path.exists(f"{OUTPUT_DIR}/{f}")]
-
-def reset():
-    state.reset()
-    return "🔄 Reset complete", state.log("Ready")
-
-# ============================================================================
-# GRADIO UI
-# ============================================================================
-
-def create_app():
-    with gr.Blocks(title="CHB BERTopic Analysis") as app:
-        gr.Markdown("""
-        # 🔬 CHB Journal BERTopic Analysis
-        **RQ5-7: Agentic AI Topic Discovery** | Mistral-7B via HuggingFace
-        """)
-        
-        with gr.Row():
-            api_status = gr.Textbox(label="API Status", value=check_api(), interactive=False)
-            reset_btn = gr.Button("🔄 Reset")
-        
-        file_input = gr.File(label="Upload Scopus CSV", file_types=[".csv"])
-        log_output = gr.Textbox(label="Log", lines=8, interactive=False)
-        
-        with gr.Tabs():
-            with gr.Tab("1️⃣ Load"):
-                load_btn = gr.Button("▶️ Load CSV", variant="primary")
-                load_out = gr.Textbox(label="Output", lines=8)
-            
-            with gr.Tab("2️⃣ Discovery"):
-                with gr.Row():
-                    disc_abs_btn = gr.Button("▶️ 2a: Abstracts")
-                    disc_title_btn = gr.Button("▶️ 2b: Titles")
-                with gr.Row():
-                    disc_abs_out = gr.Textbox(label="Abstracts", lines=8)
-                    disc_title_out = gr.Textbox(label="Titles", lines=8)
-            
-            with gr.Tab("3️⃣ Labeling"):
-                with gr.Row():
-                    label_abs_btn = gr.Button("▶️ 3a: Label Abstracts")
-                    label_title_btn = gr.Button("▶️ 3b: Label Titles")
-                with gr.Row():
-                    label_abs_out = gr.Textbox(label="Abstract Labels", lines=8)
-                    label_title_out = gr.Textbox(label="Title Labels", lines=8)
-            
-            with gr.Tab("4️⃣ Themes"):
-                with gr.Row():
-                    cons_abs_btn = gr.Button("▶️ 4a: Abstract Themes")
-                    cons_title_btn = gr.Button("▶️ 4b: Title Themes")
-                with gr.Row():
-                    cons_abs_out = gr.Textbox(label="Abstract Themes", lines=8)
-                    cons_title_out = gr.Textbox(label="Title Themes", lines=8)
-            
-            with gr.Tab("5️⃣ PAJAIS"):
-                with gr.Row():
-                    tax_abs_btn = gr.Button("▶️ 5a: Map Abstracts")
-                    tax_title_btn = gr.Button("▶️ 5b: Map Titles")
-                with gr.Row():
-                    tax_abs_out = gr.Textbox(label="Abstract Mapping", lines=8)
-                    tax_title_out = gr.Textbox(label="Title Mapping", lines=8)
-            
-            with gr.Tab("6️⃣ Compare"):
-                compare_btn = gr.Button("▶️ Compare", variant="primary")
-                compare_out = gr.Textbox(label="Comparison", lines=10)
-            
-            with gr.Tab("7️⃣ Narrative"):
-                narrative_btn = gr.Button("▶️ Generate", variant="primary")
-                narrative_out = gr.Textbox(label="Narrative", lines=15)
-            
-            with gr.Tab("📥 Downloads"):
-                download_btn = gr.Button("📥 Prepare")
-                download_files = gr.File(label="Files", file_count="multiple")
-        
-        gr.Markdown("**MDM Assignment-4** | Prasad Vijay Gade | Prof. Shailaja Jha")
-        
-        # Wire buttons
-        reset_btn.click(reset, outputs=[load_out, log_output])
-        load_btn.click(phase_1_load, inputs=[file_input], outputs=[load_out, log_output])
-        disc_abs_btn.click(phase_2_abstract, outputs=[disc_abs_out, log_output])
-        disc_title_btn.click(phase_2_title, outputs=[disc_title_out, log_output])
-        label_abs_btn.click(phase_3_abstract, outputs=[label_abs_out, log_output])
-        label_title_btn.click(phase_3_title, outputs=[label_title_out, log_output])
-        cons_abs_btn.click(phase_4_abstract, outputs=[cons_abs_out, log_output])
-        cons_title_btn.click(phase_4_title, outputs=[cons_title_out, log_output])
-        tax_abs_btn.click(phase_5_abstract, outputs=[tax_abs_out, log_output])
-        tax_title_btn.click(phase_5_title, outputs=[tax_title_out, log_output])
-        compare_btn.click(phase_6, outputs=[compare_out, log_output])
-        narrative_btn.click(phase_7, outputs=[narrative_out, log_output])
-        download_btn.click(download_outputs, outputs=[download_files])
-    
-    return app
-
-if __name__ == "__main__":
-    app = create_app()
-    app.launch(server_name="0.0.0.0")
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import gradio as gr
+
+from tools import (
+    DEFAULT_INPUT_CSV,
+    OLLAMA_MODEL,
+    OUTPUT_DIR,
+    compare_with_taxonomy,
+    consolidate_into_themes,
+    export_narrative,
+    generate_comparison_csv,
+    label_topics_with_llm,
+    load_scopus_csv,
+    run_bertopic_discovery,
+    run_full_pipeline,
+)
+
+
+CUSTOM_CSS = """
+.status-ok { color: #1b5e20; font-weight: 600; }
+.status-note { color: #37474f; }
+"""
+
+
+def _resolve_output_dir(value: str) -> Path:
+    return Path(value).resolve() if value else OUTPUT_DIR
+
+
+def ui_load(file_path: str, output_dir: str) -> str:
+    stats = load_scopus_csv(file_path, _resolve_output_dir(output_dir))
+    return json.dumps(stats, indent=2)
+
+
+def ui_discover(text_type: str, output_dir: str) -> str:
+    payload = run_bertopic_discovery(text_type, _resolve_output_dir(output_dir))
+    return json.dumps(payload, indent=2)
+
+
+def ui_label(text_type: str, output_dir: str) -> str:
+    payload = label_topics_with_llm(text_type, _resolve_output_dir(output_dir))
+    return json.dumps(payload, indent=2)
+
+
+def ui_theme(text_type: str, output_dir: str) -> str:
+    payload = consolidate_into_themes(text_type, 15, _resolve_output_dir(output_dir))
+    return json.dumps(payload, indent=2)
+
+
+def ui_taxonomy(text_type: str, output_dir: str) -> str:
+    payload = compare_with_taxonomy(text_type, _resolve_output_dir(output_dir))
+    return json.dumps(payload, indent=2)
+
+
+def ui_compare(output_dir: str) -> str:
+    payload = generate_comparison_csv(_resolve_output_dir(output_dir))
+    return json.dumps(payload, indent=2)
+
+
+def ui_narrative(output_dir: str) -> str:
+    payload = export_narrative(_resolve_output_dir(output_dir))
+    return json.dumps(payload, indent=2)
+
+
+def ui_full_pipeline(file_path: str, output_dir: str) -> str:
+    payload = run_full_pipeline(file_path=file_path, output_dir=_resolve_output_dir(output_dir))
+    return json.dumps(payload, indent=2)
+
+
+def create_interface() -> gr.Blocks:
+    with gr.Blocks(css=CUSTOM_CSS, title="CHB BERTopic App v2") as app:
+        gr.Markdown(
+            f"""
+            # CHB BERTopic App v2
+            ### DRAFT_2 rebuild: SPECTER2 + UMAP + HDBSCAN + BERTopic
+
+            **Default input:** `{DEFAULT_INPUT_CSV}`  
+            **Default output:** `{OUTPUT_DIR}`  
+            **LLM backend:** `{OLLAMA_MODEL}`
+            """
+        )
+
+        with gr.Row():
+            file_path = gr.Textbox(label="Input CSV", value=str(DEFAULT_INPUT_CSV), lines=1)
+            output_dir = gr.Textbox(label="Output directory", value=str(OUTPUT_DIR), lines=1)
+
+        full_run = gr.Button("Run full pipeline", variant="primary")
+        full_output = gr.Textbox(label="Full pipeline result", lines=18)
+        full_run.click(ui_full_pipeline, inputs=[file_path, output_dir], outputs=[full_output])
+
+        with gr.Tabs():
+            with gr.Tab("Phase 1"):
+                load_btn = gr.Button("Load corpus")
+                load_out = gr.Textbox(lines=16, label="Load output")
+                load_btn.click(ui_load, inputs=[file_path, output_dir], outputs=[load_out])
+
+            with gr.Tab("Phase 2"):
+                abs_disc_btn = gr.Button("Discover abstract topics")
+                title_disc_btn = gr.Button("Discover title topics")
+                abs_disc_out = gr.Textbox(lines=16, label="Abstract discovery")
+                title_disc_out = gr.Textbox(lines=16, label="Title discovery")
+                abs_disc_btn.click(ui_discover, inputs=[gr.State("abstract"), output_dir], outputs=[abs_disc_out])
+                title_disc_btn.click(ui_discover, inputs=[gr.State("title"), output_dir], outputs=[title_disc_out])
+
+            with gr.Tab("Phase 3"):
+                abs_label_btn = gr.Button("Label abstract topics")
+                title_label_btn = gr.Button("Label title topics")
+                abs_label_out = gr.Textbox(lines=16, label="Abstract labels")
+                title_label_out = gr.Textbox(lines=16, label="Title labels")
+                abs_label_btn.click(ui_label, inputs=[gr.State("abstract"), output_dir], outputs=[abs_label_out])
+                title_label_btn.click(ui_label, inputs=[gr.State("title"), output_dir], outputs=[title_label_out])
+
+            with gr.Tab("Phase 4"):
+                abs_theme_btn = gr.Button("Consolidate abstract themes")
+                title_theme_btn = gr.Button("Consolidate title themes")
+                abs_theme_out = gr.Textbox(lines=16, label="Abstract themes")
+                title_theme_out = gr.Textbox(lines=16, label="Title themes")
+                abs_theme_btn.click(ui_theme, inputs=[gr.State("abstract"), output_dir], outputs=[abs_theme_out])
+                title_theme_btn.click(ui_theme, inputs=[gr.State("title"), output_dir], outputs=[title_theme_out])
+
+            with gr.Tab("Phase 5"):
+                abs_tax_btn = gr.Button("Map abstract themes to PAJAIS")
+                title_tax_btn = gr.Button("Map title themes to PAJAIS")
+                abs_tax_out = gr.Textbox(lines=16, label="Abstract taxonomy")
+                title_tax_out = gr.Textbox(lines=16, label="Title taxonomy")
+                abs_tax_btn.click(ui_taxonomy, inputs=[gr.State("abstract"), output_dir], outputs=[abs_tax_out])
+                title_tax_btn.click(ui_taxonomy, inputs=[gr.State("title"), output_dir], outputs=[title_tax_out])
+
+            with gr.Tab("Phase 6"):
+                compare_btn = gr.Button("Generate comparison")
+                compare_out = gr.Textbox(lines=16, label="Comparison")
+                compare_btn.click(ui_compare, inputs=[output_dir], outputs=[compare_out])
+
+            with gr.Tab("Phase 7"):
+                narrative_btn = gr.Button("Generate narrative")
+                narrative_out = gr.Textbox(lines=16, label="Narrative result")
+                narrative_btn.click(ui_narrative, inputs=[output_dir], outputs=[narrative_out])
+
+    return app
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="CHB BERTopic draft_2 app")
+    parser.add_argument("--mode", choices=["ui", "pipeline"], default="ui")
+    parser.add_argument("--input", default=str(DEFAULT_INPUT_CSV))
+    parser.add_argument("--output-dir", default=str(OUTPUT_DIR))
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=7861)
+    args = parser.parse_args()
+
+    if args.mode == "pipeline":
+        payload = run_full_pipeline(file_path=args.input, output_dir=Path(args.output_dir))
+        print(json.dumps(payload, indent=2))
+        return
+
+    app = create_interface()
+    app.launch(server_name=args.host, server_port=args.port)
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -1,7 +1,11 @@
-gradio>=5.31.0
-huggingface-hub>=0.28.0
-sentence-transformers>=2.2.0
-scikit-learn>=1.3.0
-pandas>=2.0.0
-numpy>=1.24.0
-regex>=2023.0.0
+gradio>=4.44.0
+bertopic>=0.16.0
+umap-learn>=0.5.6
+hdbscan>=0.8.33
+transformers>=4.48.0
+adapters>=1.0.0
+scikit-learn>=1.3.0
+pandas>=2.0.0
+numpy>=1.24.0
+torch>=2.2.0
+openai>=1.30.0

tools.py

@@ -1,33 +1,48 @@
-"""
-tools.py - BERTopic Pipeline Tools for RQ5-7 (HuggingFace Deployment)
-=====================================================================
-Uses HuggingFace Inference API with free models.
-"""
+from __future__ import annotations
 
-import os
 import json
+import math
+import os
 import re
 import time
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
 import numpy as np
 import pandas as pd
-from datetime import datetime
-from sentence_transformers import SentenceTransformer
+import torch
+from adapters import AutoAdapterModel
+from bertopic import BERTopic
+from bertopic.vectorizers import ClassTfidfTransformer
+from hdbscan import HDBSCAN
+from openai import OpenAI
+from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS, CountVectorizer, TfidfVectorizer
+from sklearn.metrics import silhouette_score
 from sklearn.cluster import AgglomerativeClustering
-from sklearn.metrics.pairwise import cosine_similarity
-from huggingface_hub import InferenceClient
-
-# ============================================================================
-# CONFIGURATION
-# ============================================================================
-
-# HuggingFace free model (Mistral 7B Instruct - free with HF token)
-HF_MODEL = "mistralai/Mistral-7B-Instruct-v0.3"
-EMBEDDING_MODEL = "all-MiniLM-L6-v2"
-OUTPUT_DIR = "outputs"
-API_RETRY_DELAY = 2
+from transformers import AutoTokenizer
+from umap import UMAP
+
+
+APP_DIR = Path(__file__).resolve().parent
+PROJECT_DIR = APP_DIR.parent
+OUTPUT_DIR = PROJECT_DIR / "bertopic_outputs_7553"
+DEFAULT_INPUT_CSV = (
+    PROJECT_DIR
+    / "sources"
+    / "corpus_7553"
+    / "ComputersinHumanBehavior_TopicModelling_Export_7553_for_app.csv"
+)
+
+SPECTER2_BASE_MODEL = "allenai/specter2_base"
+SPECTER2_ADAPTER = "allenai/specter2"
+OLLAMA_BASE_URL = os.environ.get("CHB_OLLAMA_BASE_URL", "http://localhost:11434/v1")
+OLLAMA_MODEL = os.environ.get("CHB_OLLAMA_MODEL", "gemma4:e2b")
+OLLAMA_TIMEOUT_SECONDS = int(os.environ.get("CHB_OLLAMA_TIMEOUT_SECONDS", "300"))
+RANDOM_STATE = 42
 MAX_RETRIES = 3
+API_RETRY_DELAY = 2
 
-# PAJAIS 25 Categories for taxonomy mapping
 PAJAIS_CATEGORIES = [
     "Digital Innovation & Entrepreneurship",
     "Electronic Commerce",
@@ -53,10 +68,9 @@ PAJAIS_CATEGORIES = [
     "Cloud Computing",
     "Internet of Things",
     "Digital Platforms",
-    "Future of Work"
+    "Future of Work",
 ]
 
-# Boilerplate patterns to remove
 BOILERPLATE_PATTERNS = [
     r"©\s*\d{4}.*?(?:Elsevier|Ltd|Inc|reserved)",
     r"All rights reserved\.?",
@@ -64,452 +78,875 @@ BOILERPLATE_PATTERNS = [
     r"https?://\S+",
     r"\S+@\S+\.\S+",
     r"doi:?\s*\S+",
+    r"Crown Copyright.*?reserved\.?",
+    r"Published by.*?(?:Elsevier|Springer|Wiley)",
 ]
 
-# ============================================================================
-# HELPER FUNCTIONS
-# ============================================================================
-
-def _get_hf_client():
-    """Initialize HuggingFace Inference client."""
-    token = os.environ.get("HF_TOKEN", "")
-    if not token:
-        raise ValueError("HF_TOKEN environment variable not set!")
-    return InferenceClient(token=token)
-
-def _call_llm_with_retry(client, prompt, max_tokens=500):
-    """Call HuggingFace model with retry logic."""
-    last_error = None
-    for attempt in range(MAX_RETRIES):
+EXTRA_STOPWORDS = {
+    "study",
+    "paper",
+    "research",
+    "results",
+    "finding",
+    "findings",
+    "article",
+    "author",
+    "authors",
+    "based",
+    "using",
+    "used",
+    "examines",
+    "examined",
+    "investigates",
+    "investigated",
+    "analysis",
+    "implications",
+    "effect",
+    "effects",
+}
+
+TEXT_TYPE_SETTINGS = {
+    "abstract": {
+        "vectorizer_min_df": 1,
+        "vectorizer_max_df": 1.0,
+        "ngram_range": (1, 2),
+        "umap_n_neighbors": 10,
+        "umap_n_components": 5,
+        "hdbscan_min_cluster_size": 30,
+        "hdbscan_min_samples": 8,
+        "target_themes": 15,
+        "target_topics_soft_max": 55,
+    },
+    "title": {
+        "vectorizer_min_df": 1,
+        "vectorizer_max_df": 1.0,
+        "ngram_range": (1, 2),
+        "umap_n_neighbors": 8,
+        "umap_n_components": 5,
+        "hdbscan_min_cluster_size": 20,
+        "hdbscan_min_samples": 5,
+        "target_themes": 15,
+        "target_topics_soft_max": 40,
+    },
+}
+
+
+def _ensure_output_dir(output_dir: Path = OUTPUT_DIR) -> Path:
+    output_dir.mkdir(parents=True, exist_ok=True)
+    return output_dir
+
+
+def _write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+def _clean_text(text: Any) -> str:
+    value = "" if pd.isna(text) else str(text)
+    for pattern in BOILERPLATE_PATTERNS:
+        value = re.sub(pattern, " ", value, flags=re.IGNORECASE)
+    value = value.replace("\u00a9", " ")
+    value = re.sub(r"\s+", " ", value).strip()
+    return value
+
+
+def _normalize_label(value: str) -> str:
+    label = re.sub(r"\s+", " ", str(value)).strip(" -:;,.")
+    label = re.sub(r"[/|]+", " ", label)
+    label = re.sub(r"\s+", " ", label).strip()
+    return label
+
+
+def _slugify_label(value: str) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "_", value.lower())
+    return slug.strip("_")
+
+
+def _keywords_to_label(keywords: list[str], max_words: int = 4) -> str:
+    cleaned = [_normalize_label(kw.replace("_", " ")) for kw in keywords if kw.strip()]
+    if not cleaned:
+        return "Unlabeled Topic"
+
+    selected: list[str] = []
+    covered_tokens: set[str] = set()
+    for phrase in sorted(cleaned, key=lambda item: (-len(item.split()), item)):
+        tokens = [token for token in phrase.lower().split() if token]
+        if not tokens:
+            continue
+        if all(token in covered_tokens for token in tokens):
+            continue
+        selected.append(phrase)
+        covered_tokens.update(tokens)
+        if len(selected) >= 3:
+            break
+
+    if not selected:
+        selected = cleaned[:max_words]
+
+    if len(selected) == 1:
+        return selected[0].title()
+    if len(selected) == 2:
+        return f"{selected[0].title()} and {selected[1].title()}"
+    return f"{selected[0].title()}, {selected[1].title()}, and {selected[2].title()}"
+
+
+def _get_llm_client() -> OpenAI:
+    return OpenAI(
+        base_url=OLLAMA_BASE_URL,
+        api_key="ollama",
+        timeout=OLLAMA_TIMEOUT_SECONDS,
+        max_retries=0,
+    )
+
+
+def _call_llm_json(prompt: str, model: str = OLLAMA_MODEL, max_tokens: int = 300) -> dict[str, Any] | None:
+    client = _get_llm_client()
+    last_error: Exception | None = None
+    for _ in range(MAX_RETRIES):
+        try:
+            response = client.chat.completions.create(
+                model=model,
+                messages=[
+                    {
+                        "role": "system",
+                        "content": (
+                            "Return only valid JSON. Do not wrap it in markdown. "
+                            "Do not include commentary outside the JSON object."
+                        ),
+                    },
+                    {"role": "user", "content": prompt},
+                ],
+                temperature=0.2,
+                max_tokens=max_tokens,
+            )
+            text = response.choices[0].message.content.strip()
+            match = re.search(r"\{.*\}", text, flags=re.DOTALL)
+            if not match:
+                return None
+            return json.loads(match.group(0))
+        except Exception as exc:  # nosec - explicit retry path
+            last_error = exc
+            time.sleep(API_RETRY_DELAY)
+    if last_error:
+        print(f"LLM JSON call failed: {last_error}")
+    return None
+
+
+def _call_llm_text(prompt: str, model: str = OLLAMA_MODEL, max_tokens: int = 700) -> str:
+    client = _get_llm_client()
+    last_error: Exception | None = None
+    for _ in range(MAX_RETRIES):
         try:
-            response = client.text_generation(
-                prompt,
-                model=HF_MODEL,
-                max_new_tokens=max_tokens,
+            response = client.chat.completions.create(
+                model=model,
+                messages=[
+                    {"role": "system", "content": "Write clean academic text only."},
+                    {"role": "user", "content": prompt},
+                ],
                 temperature=0.3,
-                do_sample=True,
+                max_tokens=max_tokens,
             )
-            return response
-        except Exception as e:
-            last_error = e
-            print(f"HF API error (attempt {attempt + 1}/{MAX_RETRIES}): {str(e)[:100]}")
-            time.sleep(API_RETRY_DELAY * (attempt + 1))
-    raise last_error
-
-def _ensure_output_dir():
-    """Ensure output directory exists."""
-    os.makedirs(OUTPUT_DIR, exist_ok=True)
-    return OUTPUT_DIR
-
-def _clean_text(text):
-    """Remove boilerplate from text."""
-    result = str(text) if text else ""
-    for pattern in BOILERPLATE_PATTERNS:
-        result = re.sub(pattern, "", result, flags=re.IGNORECASE)
-    return result.strip()
-
-def _split_sentences(text):
-    """Split text into sentences."""
-    sentences = re.split(r'(?<=[.!?])\s+', str(text))
-    return [s.strip() for s in sentences if len(s.strip()) > 20]
-
-# ============================================================================
-# TOOL 1: load_scopus_csv
-# ============================================================================
-
-def load_scopus_csv(file_path: str) -> dict:
-    """Load and clean Scopus CSV data."""
-    _ensure_output_dir()
-    
-    df = pd.read_csv(file_path)
+            return response.choices[0].message.content.strip()
+        except Exception as exc:  # nosec - explicit retry path
+            last_error = exc
+            time.sleep(API_RETRY_DELAY)
+    if last_error:
+        print(f"LLM text call failed: {last_error}")
+    return ""
+
+
+class Specter2Embedder:
+    def __init__(
+        self,
+        model_name: str = SPECTER2_BASE_MODEL,
+        adapter_name: str = SPECTER2_ADAPTER,
+        batch_size: int = 8,
+        max_length: int = 512,
+    ) -> None:
+        self.model_name = model_name
+        self.adapter_name = adapter_name
+        self.batch_size = batch_size
+        self.max_length = max_length
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.tokenizer: AutoTokenizer | None = None
+        self.model: AutoAdapterModel | None = None
+
+    def _load(self) -> None:
+        if self.tokenizer is not None and self.model is not None:
+            return
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+        self.model = AutoAdapterModel.from_pretrained(self.model_name)
+        self.model.load_adapter(self.adapter_name, source="hf", load_as="specter2", set_active=True)
+        self.model.set_active_adapters("specter2")
+        self.model.to(self.device)
+        self.model.eval()
+
+    def encode(self, texts: list[str], cache_path: Path | None = None) -> np.ndarray:
+        if cache_path and cache_path.exists():
+            cached = np.load(cache_path)
+            if len(cached) == len(texts):
+                return cached
+
+        self._load()
+        assert self.tokenizer is not None
+        assert self.model is not None
+
+        embeddings: list[np.ndarray] = []
+        total_batches = math.ceil(len(texts) / self.batch_size)
+        for batch_index in range(total_batches):
+            start = batch_index * self.batch_size
+            end = start + self.batch_size
+            batch = texts[start:end]
+            encoded = self.tokenizer(
+                batch,
+                padding=True,
+                truncation=True,
+                max_length=self.max_length,
+                return_token_type_ids=False,
+                return_tensors="pt",
+            )
+            encoded = {key: value.to(self.device) for key, value in encoded.items()}
+            with torch.no_grad():
+                outputs = self.model(**encoded)
+                batch_embeddings = outputs.last_hidden_state[:, 0, :]
+                batch_embeddings = torch.nn.functional.normalize(batch_embeddings, p=2, dim=1)
+            embeddings.append(batch_embeddings.cpu().numpy().astype(np.float32))
+            print(f"SPECTER2 batch {batch_index + 1}/{total_batches} complete")
+
+        matrix = np.vstack(embeddings)
+        if cache_path:
+            np.save(cache_path, matrix)
+        return matrix
+
+
+def _prepare_embedding_text(df: pd.DataFrame, text_type: str, sep_token: str) -> list[str]:
+    if text_type == "title":
+        return df["Title_Clean"].tolist()
+    return [
+        f"{title}{sep_token}{abstract}".strip()
+        for title, abstract in zip(df["Title_Clean"], df["Abstract_Clean"], strict=True)
+    ]
+
+
+def _load_clean_papers(output_dir: Path = OUTPUT_DIR) -> pd.DataFrame:
+    path = output_dir / "papers_clean.csv"
+    if not path.exists():
+        raise FileNotFoundError("Run load_scopus_csv first.")
+    return pd.read_csv(path, keep_default_na=False)
+
+
+def load_scopus_csv(file_path: str, output_dir: Path = OUTPUT_DIR) -> dict[str, Any]:
+    output_dir = _ensure_output_dir(output_dir)
+    df = pd.read_csv(file_path, keep_default_na=False)
+    df["Title_Clean"] = df["Title"].apply(_clean_text)
     df["Abstract_Clean"] = df["Abstract"].apply(_clean_text)
-    
-    all_sentences = []
-    paper_ids = []
-    
-    for idx, row in df.iterrows():
-        sentences = _split_sentences(row["Abstract_Clean"])
-        all_sentences.extend(sentences)
-        paper_ids.extend([idx] * len(sentences))
-    
-    sentences_df = pd.DataFrame({"sentence": all_sentences, "paper_id": paper_ids})
-    sentences_df.to_csv(f"{OUTPUT_DIR}/sentences.csv", index=False)
-    df.to_csv(f"{OUTPUT_DIR}/papers_clean.csv", index=False)
-    
+    df["Author Keywords"] = df["Author Keywords"].fillna("").astype(str)
+    df["Cited by"] = pd.to_numeric(df["Cited by"], errors="coerce").fillna(0).astype(int)
+    df["Year"] = pd.to_numeric(df["Year"], errors="coerce").astype(int)
+    if "Sr No" not in df.columns:
+        df.insert(0, "Sr No", range(1, len(df) + 1))
+    df.to_csv(output_dir / "papers_clean.csv", index=False, encoding="utf-8-sig")
+
     stats = {
-        "total_papers": len(df),
-        "total_sentences": len(all_sentences),
+        "total_papers": int(len(df)),
+        "year_range": f"{int(df['Year'].min())} - {int(df['Year'].max())}",
         "columns": list(df.columns),
+        "sample_titles": df["Title_Clean"].head(3).tolist(),
         "sample_abstracts": df["Abstract_Clean"].head(3).tolist(),
-        "year_range": f"{df['Year'].min()} - {df['Year'].max()}" if "Year" in df.columns else "N/A",
-        "timestamp": datetime.now().isoformat()
+        "timestamp": pd.Timestamp.utcnow().isoformat(),
     }
-    
-    with open(f"{OUTPUT_DIR}/data_stats.json", "w") as f:
-        json.dump(stats, f, indent=2)
-    
+    _write_json(output_dir / "data_stats.json", stats)
     return stats
 
-# ============================================================================
-# TOOL 2: run_bertopic_discovery
-# ============================================================================
-
-def run_bertopic_discovery(column_name: str = "Abstract") -> dict:
-    """Embed and cluster texts into topics."""
-    _ensure_output_dir()
-    
-    papers_df = pd.read_csv(f"{OUTPUT_DIR}/papers_clean.csv")
-    
-    if column_name == "Title":
-        texts = papers_df["Title"].apply(_clean_text).tolist()
-        text_type = "title"
-    else:
-        sentences_df = pd.read_csv(f"{OUTPUT_DIR}/sentences.csv")
-        texts = sentences_df["sentence"].tolist()
-        text_type = "abstract"
-    
-    print(f"Embedding {len(texts)} texts...")
-    model = SentenceTransformer(EMBEDDING_MODEL)
-    embeddings = model.encode(texts, show_progress_bar=True, normalize_embeddings=True)
-    
-    np.save(f"{OUTPUT_DIR}/embeddings_{text_type}.npy", embeddings)
-    
-    print("Clustering...")
-    clustering = AgglomerativeClustering(
-        n_clusters=None,
-        distance_threshold=0.7,
+
+def _build_topic_model(text_type: str) -> BERTopic:
+    settings = TEXT_TYPE_SETTINGS[text_type]
+    stop_words = sorted(ENGLISH_STOP_WORDS.union(EXTRA_STOPWORDS))
+    vectorizer_model = CountVectorizer(
+        stop_words=stop_words,
+        ngram_range=settings["ngram_range"],
+        min_df=settings["vectorizer_min_df"],
+        max_df=settings["vectorizer_max_df"],
+    )
+    umap_model = UMAP(
+        n_neighbors=settings["umap_n_neighbors"],
+        n_components=settings["umap_n_components"],
+        min_dist=0.0,
         metric="cosine",
-        linkage="average"
+        random_state=RANDOM_STATE,
+        low_memory=True,
     )
-    labels = clustering.fit_predict(embeddings)
-    
-    unique_labels = set(labels)
-    topics = {}
-    
-    for label in unique_labels:
-        if label == -1:
-            continue
-        indices = [i for i, l in enumerate(labels) if l == label]
-        cluster_embeddings = embeddings[indices]
-        centroid = np.mean(cluster_embeddings, axis=0)
-        similarities = cosine_similarity([centroid], cluster_embeddings)[0]
-        top_indices = np.argsort(similarities)[-5:][::-1]
-        representative_texts = [texts[indices[i]] for i in top_indices]
-        
-        topics[int(label)] = {
-            "size": len(indices),
-            "representative_texts": representative_texts,
-        }
-    
-    topics = dict(sorted(topics.items(), key=lambda x: x[1]["size"], reverse=True))
-    
-    result = {
-        "column": column_name,
-        "text_type": text_type,
-        "total_texts": len(texts),
-        "num_topics": len(topics),
-        "topics": topics,
-        "timestamp": datetime.now().isoformat()
-    }
-    
-    with open(f"{OUTPUT_DIR}/summaries_{text_type}.json", "w") as f:
-        json.dump(result, f, indent=2, default=str)
-    
-    pd.DataFrame({"text": texts, "topic": labels}).to_csv(
-        f"{OUTPUT_DIR}/topic_assignments_{text_type}.csv", index=False
+    hdbscan_model = HDBSCAN(
+        min_cluster_size=settings["hdbscan_min_cluster_size"],
+        min_samples=settings["hdbscan_min_samples"],
+        metric="euclidean",
+        cluster_selection_method="eom",
+        prediction_data=True,
+    )
+    return BERTopic(
+        umap_model=umap_model,
+        hdbscan_model=hdbscan_model,
+        vectorizer_model=vectorizer_model,
+        ctfidf_model=ClassTfidfTransformer(reduce_frequent_words=True),
+        top_n_words=10,
+        verbose=True,
+        calculate_probabilities=False,
+        low_memory=True,
+        min_topic_size=settings["hdbscan_min_cluster_size"],
     )
-    
-    return {
-        "num_topics": len(topics),
-        "total_texts": len(texts),
-        "top_10_topics": {k: {"size": v["size"], "sample": v["representative_texts"][0][:100]} 
-                          for k, v in list(topics.items())[:10]}
-    }
 
-# ============================================================================
-# TOOL 3: label_topics_with_llm
-# ============================================================================
-
-def label_topics_with_llm(text_type: str = "abstract") -> dict:
-    """Generate topic labels using HuggingFace LLM."""
-    _ensure_output_dir()
-    client = _get_hf_client()
-    
-    with open(f"{OUTPUT_DIR}/summaries_{text_type}.json", "r") as f:
-        data = json.load(f)
-    
-    topics = data["topics"]
-    labeled_topics = {}
-    
-    # Label top 20 topics to stay within free tier limits
-    topics_to_label = list(topics.items())[:20]
-    
-    for idx, (topic_id, topic_data) in enumerate(topics_to_label):
-        representative_texts = topic_data["representative_texts"]
-        texts_str = "\n".join([f"- {t[:150]}" for t in representative_texts[:3]])
-        
-        prompt = f"""<s>[INST] Given these representative sentences from academic abstracts in "Computers in Human Behavior" journal, generate a concise topic label (3-5 words).
-
-Sentences:
-{texts_str}
-
-Respond with ONLY a JSON object: {{"label": "Your Topic Label"}} [/INST]"""
 
-        try:
-            response = _call_llm_with_retry(client, prompt, max_tokens=50)
-            json_match = re.search(r'\{[^}]+\}', response)
-            if json_match:
-                label_data = json.loads(json_match.group())
-                label = label_data.get("label", f"Topic {topic_id}")
-            else:
-                label = f"Topic {topic_id}"
-        except Exception as e:
-            print(f"Error labeling topic {topic_id}: {e}")
-            label = f"Topic {topic_id}"
-        
-        labeled_topics[topic_id] = {
+def _compute_topic_diversity(topic_model: BERTopic, top_n_words: int = 10) -> float:
+    words: list[str] = []
+    for topic_id, values in topic_model.get_topics().items():
+        if topic_id == -1:
+            continue
+        words.extend(word for word, _ in values[:top_n_words])
+    if not words:
+        return 0.0
+    return len(set(words)) / len(words)
+
+
+def _compute_silhouette(embeddings: np.ndarray, topics: list[int]) -> float | None:
+    valid_indices = [idx for idx, topic_id in enumerate(topics) if topic_id != -1]
+    valid_labels = [topics[idx] for idx in valid_indices]
+    if len(set(valid_labels)) < 2 or len(valid_indices) < 200:
+        return None
+    sample_size = min(1500, len(valid_indices))
+    sampled = valid_indices[:sample_size]
+    try:
+        return float(
+            silhouette_score(
+                embeddings[sampled],
+                [topics[idx] for idx in sampled],
+                metric="cosine",
+            )
+        )
+    except Exception:
+        return None
+
+
+def _topic_keywords(topic_model: BERTopic, topic_id: int, top_n_words: int = 8) -> list[str]:
+    return [word for word, _ in topic_model.get_topic(topic_id)[:top_n_words]]
+
+
+def _representative_docs_map(topic_model: BERTopic) -> dict[int, list[str]]:
+    raw_docs = topic_model.get_representative_docs() or {}
+    return {int(topic_id): [str(doc) for doc in docs] for topic_id, docs in raw_docs.items()}
+
+
+def _relabel_topics_with_fallback(topic_model: BERTopic, text_type: str, output_dir: Path) -> dict[str, Any]:
+    topic_info = topic_model.get_topic_info()
+    representative_docs = _representative_docs_map(topic_model)
+    labels: dict[str, Any] = {}
+
+    for _, row in topic_info.iterrows():
+        topic_id = int(row["Topic"])
+        if topic_id == -1:
+            continue
+        keywords = _topic_keywords(topic_model, topic_id)
+        base_label = _keywords_to_label(keywords)
+        docs = representative_docs.get(topic_id, [])[:3]
+        prompt = f"""
+Create one concise human-readable research topic label for a BERTopic cluster from Computers in Human Behavior.
+
+Rules:
+1. 3 to 6 words
+2. academic and specific
+3. do not output generic labels like "topic", "cluster", "research topic", or repeated stopwords
+4. prefer phenomenon or domain wording over raw keywords
+5. return the label only, no explanation
+
+Topic keywords: {", ".join(keywords)}
+Representative texts:
+{chr(10).join("- " + doc[:500] for doc in docs)}
+"""
+        response_text = _call_llm_text(prompt, max_tokens=40)
+        label = base_label
+        confidence = 0.5
+        if response_text:
+            candidate = _normalize_label(response_text.splitlines()[0])
+            if (
+                candidate
+                and not re.fullmatch(r"topic\s*\d+", candidate, flags=re.IGNORECASE)
+                and " and of " not in candidate.lower()
+            ):
+                label = candidate
+                confidence = 0.8
+
+        labels[str(topic_id)] = {
             "label": label,
-            "size": topic_data["size"],
-            "representative_texts": representative_texts
+            "confidence": confidence,
+            "topic_size": int(row["Count"]),
+            "keywords": keywords,
+            "representative_texts": docs,
         }
-        
-        print(f"Topic {topic_id}: {label}")
-        time.sleep(1)  # Rate limit
-    
-    result = {
+
+    payload = {
         "text_type": text_type,
-        "num_labeled": len(labeled_topics),
-        "topics": labeled_topics,
-        "timestamp": datetime.now().isoformat()
-    }
-    
-    with open(f"{OUTPUT_DIR}/labels_{text_type}.json", "w") as f:
-        json.dump(result, f, indent=2)
-    
-    return {
-        "num_labeled": len(labeled_topics),
-        "sample_labels": {k: v["label"] for k, v in list(labeled_topics.items())[:10]}
+        "num_labeled": len(labels),
+        "topics": labels,
+        "timestamp": pd.Timestamp.utcnow().isoformat(),
     }
+    _write_json(output_dir / f"labels_{text_type}.json", payload)
+    return payload
 
-# ============================================================================
-# TOOL 4: consolidate_into_themes
-# ============================================================================
-
-def consolidate_into_themes(text_type: str = "abstract", target_themes: int = 15) -> dict:
-    """Merge similar topics into broader themes."""
-    _ensure_output_dir()
-    client = _get_hf_client()
-    
-    with open(f"{OUTPUT_DIR}/labels_{text_type}.json", "r") as f:
-        data = json.load(f)
-    
-    topics = data["topics"]
-    labels_list = [{"id": k, "label": v["label"], "size": v["size"]} for k, v in topics.items()]
-    labels_str = "\n".join([f"- {t['label']} (ID:{t['id']}, {t['size']} items)" for t in labels_list[:15]])
-    
-    prompt = f"""<s>[INST] Group these research topics into {target_themes} broader themes:
-
-{labels_str}
-
-Respond with ONLY a JSON object mapping theme names to topic IDs:
-{{"Theme Name": ["id1", "id2"], "Another Theme": ["id3"]}} [/INST]"""
 
-    try:
-        response = _call_llm_with_retry(client, prompt, max_tokens=500)
-        json_match = re.search(r'\{[^{}]*(?:\[[^\]]*\][^{}]*)*\}', response, re.DOTALL)
-        if json_match:
-            theme_groups = json.loads(json_match.group())
-        else:
-            theme_groups = {topics[k]["label"]: [k] for k in list(topics.keys())[:target_themes]}
-    except Exception as e:
-        print(f"Consolidation error: {e}")
-        theme_groups = {topics[k]["label"]: [k] for k in list(topics.keys())[:target_themes]}
-    
-    themes = {}
+def _fallback_theme_groups(labels_payload: dict[str, Any], target_themes: int) -> dict[str, list[str]]:
+    rows = []
+    for topic_id, info in labels_payload["topics"].items():
+        corpus_text = " ".join([info["label"]] + info["keywords"][:6])
+        rows.append((topic_id, corpus_text, info["topic_size"]))
+    texts = [item[1] for item in rows]
+    ids = [item[0] for item in rows]
+
+    if len(texts) <= target_themes:
+        return {labels_payload["topics"][topic_id]["label"]: [topic_id] for topic_id in ids}
+
+    vectorizer = TfidfVectorizer(stop_words="english", ngram_range=(1, 2))
+    matrix = vectorizer.fit_transform(texts)
+    clustering = AgglomerativeClustering(n_clusters=target_themes, metric="cosine", linkage="average")
+    cluster_ids = clustering.fit_predict(matrix.toarray())
+
+    groups: dict[int, list[str]] = {}
+    for topic_id, cluster_id in zip(ids, cluster_ids, strict=True):
+        groups.setdefault(int(cluster_id), []).append(topic_id)
+
+    theme_groups: dict[str, list[str]] = {}
+    for cluster_id, topic_ids in groups.items():
+        all_keywords = Counter()
+        for topic_id in topic_ids:
+            all_keywords.update(labels_payload["topics"][topic_id]["keywords"][:5])
+        label = _keywords_to_label([word for word, _ in all_keywords.most_common(4)])
+        theme_groups[f"{label} Theme {cluster_id + 1}"] = topic_ids
+    return theme_groups
+
+
+def consolidate_into_themes(
+    text_type: str,
+    target_themes: int = 15,
+    output_dir: Path = OUTPUT_DIR,
+) -> dict[str, Any]:
+    output_dir = _ensure_output_dir(output_dir)
+    labels_payload = _read_json(output_dir / f"labels_{text_type}.json")
+    topics = labels_payload["topics"]
+    topic_rows = []
+    for topic_id, info in topics.items():
+        topic_rows.append(
+            {
+                "topic_id": topic_id,
+                "label": info["label"],
+                "size": info["topic_size"],
+                "keywords": ", ".join(info["keywords"][:6]),
+            }
+        )
+    topic_rows = sorted(topic_rows, key=lambda item: item["size"], reverse=True)
+    prompt = f"""
+Group these CHB topic labels into approximately {target_themes} broader human-readable themes.
+
+Topics:
+{chr(10).join(f"- {row['topic_id']}: {row['label']} | size={row['size']} | keywords={row['keywords']}" for row in topic_rows)}
+
+Return JSON only:
+{{
+  "themes": [
+    {{
+      "theme_name": "Human-readable theme",
+      "topic_ids": ["0", "4", "18"]
+    }}
+  ]
+}}
+"""
+    response = _call_llm_json(prompt, max_tokens=1200)
+    if response and isinstance(response.get("themes"), list):
+        theme_groups = {
+            _normalize_label(item["theme_name"]): [str(topic_id) for topic_id in item["topic_ids"]]
+            for item in response["themes"]
+            if item.get("theme_name") and item.get("topic_ids")
+        }
+    else:
+        theme_groups = _fallback_theme_groups(labels_payload, target_themes)
+
+    themes: dict[str, Any] = {}
     for theme_name, topic_ids in theme_groups.items():
-        total_size = sum(topics.get(str(tid), {}).get("size", 0) for tid in topic_ids)
-        member_labels = [topics.get(str(tid), {}).get("label", "Unknown") for tid in topic_ids]
+        member_labels = [topics[str(topic_id)]["label"] for topic_id in topic_ids if str(topic_id) in topics]
+        member_keywords = []
+        total_size = 0
+        for topic_id in topic_ids:
+            info = topics.get(str(topic_id))
+            if not info:
+                continue
+            member_keywords.extend(info["keywords"][:4])
+            total_size += int(info["topic_size"])
         themes[theme_name] = {
-            "topic_ids": topic_ids,
+            "topic_ids": [str(topic_id) for topic_id in topic_ids],
+            "member_labels": member_labels,
+            "keywords": [word for word, _ in Counter(member_keywords).most_common(8)],
             "total_size": total_size,
-            "member_labels": member_labels
         }
-    
-    result = {
+
+    payload = {
         "text_type": text_type,
         "num_themes": len(themes),
-        "themes": themes,
-        "timestamp": datetime.now().isoformat()
+        "themes": dict(sorted(themes.items(), key=lambda item: item[1]["total_size"], reverse=True)),
+        "timestamp": pd.Timestamp.utcnow().isoformat(),
+    }
+    _write_json(output_dir / f"themes_{text_type}.json", payload)
+    return {
+        "num_themes": payload["num_themes"],
+        "themes": {name: info["total_size"] for name, info in payload["themes"].items()},
     }
-    
-    with open(f"{OUTPUT_DIR}/themes_{text_type}.json", "w") as f:
-        json.dump(result, f, indent=2)
-    
-    return {"num_themes": len(themes), "themes": {k: v["total_size"] for k, v in themes.items()}}
-
-# ============================================================================
-# TOOL 5: compare_with_taxonomy
-# ============================================================================
-
-def compare_with_taxonomy(text_type: str = "abstract") -> dict:
-    """Map themes to PAJAIS taxonomy."""
-    _ensure_output_dir()
-    client = _get_hf_client()
-    
-    with open(f"{OUTPUT_DIR}/themes_{text_type}.json", "r") as f:
-        data = json.load(f)
-    
-    themes = data["themes"]
-    taxonomy_map = {}
-    pajais_str = "\n".join([f"{i+1}. {cat}" for i, cat in enumerate(PAJAIS_CATEGORIES)])
-    
-    for theme_name, theme_data in themes.items():
-        prompt = f"""<s>[INST] Does this research theme map to a PAJAIS IS category or is it NOVEL?
-
-Theme: "{theme_name}"
-Topics: {', '.join(theme_data['member_labels'][:3])}
-
-PAJAIS Categories:
-{pajais_str}
 
-Respond with ONLY JSON: {{"mapping_type": "MAPPED" or "NOVEL", "pajais_category": "category name or null", "reasoning": "brief reason"}} [/INST]"""
 
-        try:
-            response = _call_llm_with_retry(client, prompt, max_tokens=150)
-            json_match = re.search(r'\{[^}]+\}', response, re.DOTALL)
-            if json_match:
-                mapping = json.loads(json_match.group())
-            else:
-                mapping = {"mapping_type": "NOVEL", "pajais_category": None, "reasoning": "Parse error"}
-        except Exception as e:
-            mapping = {"mapping_type": "NOVEL", "pajais_category": None, "reasoning": str(e)[:50]}
-        
+def _fallback_taxonomy(theme_name: str, keywords: list[str]) -> dict[str, Any]:
+    tokens = set(re.findall(r"[a-z]+", f"{theme_name} {' '.join(keywords)}".lower()))
+    category_tokens = {
+        category: set(re.findall(r"[a-z]+", category.lower()))
+        for category in PAJAIS_CATEGORIES
+    }
+    scored = []
+    for category, candidate_tokens in category_tokens.items():
+        overlap = len(tokens & candidate_tokens)
+        scored.append((category, overlap))
+    scored.sort(key=lambda item: item[1], reverse=True)
+    best_category, score = scored[0]
+    if score == 0:
+        return {
+            "mapping_type": "NOVEL",
+            "pajais_category": None,
+            "confidence": 0.45,
+            "reasoning": "No direct lexical overlap with PAJAIS categories.",
+        }
+    return {
+        "mapping_type": "MAPPED",
+        "pajais_category": best_category,
+        "confidence": round(min(0.85, 0.45 + 0.10 * score), 2),
+        "reasoning": "Fallback lexical match against PAJAIS categories.",
+    }
+
+
+def compare_with_taxonomy(text_type: str, output_dir: Path = OUTPUT_DIR) -> dict[str, Any]:
+    output_dir = _ensure_output_dir(output_dir)
+    payload = _read_json(output_dir / f"themes_{text_type}.json")
+    themes = payload["themes"]
+    taxonomy_map: dict[str, Any] = {}
+    pajais_str = "\n".join(f"{index + 1}. {category}" for index, category in enumerate(PAJAIS_CATEGORIES))
+
+    for theme_name, info in themes.items():
+        prompt = f"""
+Map this CHB research theme to the PAJAIS 25-category taxonomy.
+
+Theme: {theme_name}
+Member labels: {", ".join(info["member_labels"][:8])}
+Keywords: {", ".join(info["keywords"][:8])}
+
+PAJAIS categories:
+{pajais_str}
+
+Return JSON only:
+{{
+  "mapping_type": "MAPPED or NOVEL",
+  "pajais_category": "Category name or null",
+  "confidence": 0.0,
+  "reasoning": "One short sentence"
+}}
+"""
+        response = _call_llm_json(prompt, max_tokens=220)
+        mapping = response if response else _fallback_taxonomy(theme_name, info["keywords"])
         taxonomy_map[theme_name] = {
-            **mapping,
-            "size": theme_data["total_size"],
-            "member_labels": theme_data["member_labels"]
+            "mapping_type": mapping["mapping_type"],
+            "pajais_category": mapping.get("pajais_category"),
+            "confidence": float(mapping.get("confidence", 0.5)),
+            "reasoning": mapping.get("reasoning", ""),
+            "size": int(info["total_size"]),
+            "member_labels": info["member_labels"],
+            "keywords": info["keywords"],
+            "topic_ids": info["topic_ids"],
         }
-        
-        print(f"{theme_name}: {mapping.get('mapping_type')} -> {mapping.get('pajais_category')}")
-        time.sleep(1)
-    
-    mapped_count = sum(1 for v in taxonomy_map.values() if v["mapping_type"] == "MAPPED")
+
+    mapped_count = sum(1 for item in taxonomy_map.values() if item["mapping_type"] == "MAPPED")
     novel_count = len(taxonomy_map) - mapped_count
-    
     result = {
         "text_type": text_type,
         "total_themes": len(taxonomy_map),
         "mapped_count": mapped_count,
         "novel_count": novel_count,
         "taxonomy_map": taxonomy_map,
-        "timestamp": datetime.now().isoformat()
+        "timestamp": pd.Timestamp.utcnow().isoformat(),
     }
-    
-    with open(f"{OUTPUT_DIR}/taxonomy_map_{text_type}.json", "w") as f:
-        json.dump(result, f, indent=2)
-    
+    _write_json(output_dir / f"taxonomy_map_{text_type}.json", result)
     return {
         "mapped_count": mapped_count,
         "novel_count": novel_count,
-        "mapped_themes": [k for k, v in taxonomy_map.items() if v["mapping_type"] == "MAPPED"],
-        "novel_themes": [k for k, v in taxonomy_map.items() if v["mapping_type"] == "NOVEL"]
+        "mapped_themes": [name for name, info in taxonomy_map.items() if info["mapping_type"] == "MAPPED"],
+        "novel_themes": [name for name, info in taxonomy_map.items() if info["mapping_type"] == "NOVEL"],
     }
 
-# ============================================================================
-# TOOL 6: generate_comparison_csv
-# ============================================================================
-
-def generate_comparison_csv() -> dict:
-    """Compare abstract themes vs title themes."""
-    _ensure_output_dir()
-    
-    with open(f"{OUTPUT_DIR}/taxonomy_map_abstract.json", "r") as f:
-        abstract_data = json.load(f)
-    
-    with open(f"{OUTPUT_DIR}/taxonomy_map_title.json", "r") as f:
-        title_data = json.load(f)
-    
-    abstract_themes = set(abstract_data["taxonomy_map"].keys())
-    title_themes = set(title_data["taxonomy_map"].keys())
-    
-    both = abstract_themes & title_themes
-    abstract_only = abstract_themes - title_themes
-    title_only = title_themes - abstract_themes
-    
+
+def generate_comparison_csv(output_dir: Path = OUTPUT_DIR) -> dict[str, Any]:
+    output_dir = _ensure_output_dir(output_dir)
+    abstract_tax = _read_json(output_dir / "taxonomy_map_abstract.json")
+    title_tax = _read_json(output_dir / "taxonomy_map_title.json")
+    abstract_themes = abstract_tax["taxonomy_map"]
+    title_themes = title_tax["taxonomy_map"]
+
+    normalized_abstract = {_slugify_label(name): name for name in abstract_themes}
+    normalized_title = {_slugify_label(name): name for name in title_themes}
+    overlap_keys = set(normalized_abstract) & set(normalized_title)
+
     rows = []
-    for theme in abstract_themes | title_themes:
-        abs_info = abstract_data["taxonomy_map"].get(theme, {})
-        title_info = title_data["taxonomy_map"].get(theme, {})
-        rows.append({
-            "Theme": theme,
-            "In_Abstracts": "Yes" if theme in abstract_themes else "No",
-            "In_Titles": "Yes" if theme in title_themes else "No",
-            "Abstract_Size": abs_info.get("size", 0),
-            "Title_Size": title_info.get("size", 0),
-            "Abstract_Mapping": abs_info.get("mapping_type", "N/A"),
-            "Title_Mapping": title_info.get("mapping_type", "N/A"),
-            "PAJAIS_Category": abs_info.get("pajais_category") or title_info.get("pajais_category") or "NOVEL"
-        })
-    
-    comparison_df = pd.DataFrame(rows).sort_values("Abstract_Size", ascending=False)
-    comparison_df.to_csv(f"{OUTPUT_DIR}/comparison.csv", index=False)
-    
+    for norm_key in sorted(set(normalized_abstract) | set(normalized_title)):
+        abstract_name = normalized_abstract.get(norm_key)
+        title_name = normalized_title.get(norm_key)
+        abstract_info = abstract_themes.get(abstract_name, {})
+        title_info = title_themes.get(title_name, {})
+        rows.append(
+            {
+                "Normalized_Theme_Key": norm_key,
+                "Abstract_Theme": abstract_name or "",
+                "Title_Theme": title_name or "",
+                "In_Abstracts": "Yes" if abstract_name else "No",
+                "In_Titles": "Yes" if title_name else "No",
+                "Abstract_Size": abstract_info.get("size", 0),
+                "Title_Size": title_info.get("size", 0),
+                "Abstract_Mapping": abstract_info.get("mapping_type", ""),
+                "Title_Mapping": title_info.get("mapping_type", ""),
+                "Abstract_PAJAIS": abstract_info.get("pajais_category", ""),
+                "Title_PAJAIS": title_info.get("pajais_category", ""),
+            }
+        )
+
+    comparison_df = pd.DataFrame(rows).sort_values(
+        ["In_Abstracts", "In_Titles", "Abstract_Size", "Title_Size"],
+        ascending=[False, False, False, False],
+    )
+    comparison_df.to_csv(output_dir / "comparison.csv", index=False, encoding="utf-8-sig")
+
     result = {
-        "total_unique_themes": len(abstract_themes | title_themes),
-        "themes_in_both": len(both),
-        "abstract_only": len(abstract_only),
-        "title_only": len(title_only),
-        "timestamp": datetime.now().isoformat()
+        "total_unique_themes": int(len(rows)),
+        "themes_in_both": int(len(overlap_keys)),
+        "abstract_only": int(len(set(normalized_abstract) - overlap_keys)),
+        "title_only": int(len(set(normalized_title) - overlap_keys)),
+        "abstract_only_themes": [normalized_abstract[key] for key in sorted(set(normalized_abstract) - overlap_keys)],
+        "title_only_themes": [normalized_title[key] for key in sorted(set(normalized_title) - overlap_keys)],
+        "themes_in_both_labels": [
+            {
+                "normalized_key": key,
+                "abstract_theme": normalized_abstract[key],
+                "title_theme": normalized_title[key],
+            }
+            for key in sorted(overlap_keys)
+        ],
+        "timestamp": pd.Timestamp.utcnow().isoformat(),
     }
-    
-    with open(f"{OUTPUT_DIR}/comparison_summary.json", "w") as f:
-        json.dump(result, f, indent=2)
-    
+    _write_json(output_dir / "comparison_summary.json", result)
     return result
 
-# ============================================================================
-# TOOL 7: export_narrative
-# ============================================================================
-
-def export_narrative() -> dict:
-    """Generate Section 7 narrative."""
-    _ensure_output_dir()
-    client = _get_hf_client()
-    
-    with open(f"{OUTPUT_DIR}/data_stats.json", "r") as f:
-        stats = json.load(f)
-    
-    with open(f"{OUTPUT_DIR}/taxonomy_map_title.json", "r") as f:
-        title_tax = json.load(f)
-    
-    with open(f"{OUTPUT_DIR}/comparison_summary.json", "r") as f:
-        comparison = json.load(f)
-    
-    prompt = f"""<s>[INST] Write a 300-word academic methodology section for analyzing "Computers in Human Behavior" journal.
-
-Data: {stats['total_papers']} papers, {stats['total_sentences']} sentences, years {stats['year_range']}
-Themes: {title_tax['total_themes']} discovered, {title_tax['mapped_count']} MAPPED to PAJAIS, {title_tax['novel_count']} NOVEL
-Comparison: {comparison['themes_in_both']} themes in both abstracts/titles, {comparison['abstract_only']} abstract-only, {comparison['title_only']} title-only
-
-Include: methodology (BERTopic, sentence-transformers), key findings, PAJAIS mapping, limitations. [/INST]"""
 
-    try:
-        narrative = _call_llm_with_retry(client, prompt, max_tokens=800)
-    except Exception as e:
-        narrative = f"Narrative generation failed: {e}"
-    
-    with open(f"{OUTPUT_DIR}/narrative.txt", "w", encoding="utf-8") as f:
-        f.write(narrative)
-    
-    return {"word_count": len(narrative.split()), "preview": narrative[:500]}
-
-
-if __name__ == "__main__":
-    print("Tools module loaded. Available functions:")
-    print("1. load_scopus_csv(file_path)")
-    print("2. run_bertopic_discovery(column_name)")
-    print("3. label_topics_with_llm(text_type)")
-    print("4. consolidate_into_themes(text_type)")
-    print("5. compare_with_taxonomy(text_type)")
-    print("6. generate_comparison_csv()")
-    print("7. export_narrative()")
+def export_narrative(output_dir: Path = OUTPUT_DIR) -> dict[str, Any]:
+    output_dir = _ensure_output_dir(output_dir)
+    stats = _read_json(output_dir / "data_stats.json")
+    abstract_tax = _read_json(output_dir / "taxonomy_map_abstract.json")
+    title_tax = _read_json(output_dir / "taxonomy_map_title.json")
+    comparison = _read_json(output_dir / "comparison_summary.json")
+
+    prompt = f"""
+Write a concise 500-word academic results narrative for the CHB BERTopic draft_2 pipeline.
+
+Facts to use:
+- Corpus size: {stats['total_papers']} papers
+- Year range: {stats['year_range']}
+- Abstract themes: {abstract_tax['total_themes']}
+- Abstract mapped themes: {abstract_tax['mapped_count']}
+- Abstract novel themes: {abstract_tax['novel_count']}
+- Title themes: {title_tax['total_themes']}
+- Title mapped themes: {title_tax['mapped_count']}
+- Title novel themes: {title_tax['novel_count']}
+- Theme overlap between abstract and title sets: {comparison['themes_in_both']}
+
+Requirements:
+1. Mention SPECTER2, UMAP, HDBSCAN, BERTopic, and PAJAIS.
+2. Stay factual.
+3. Do not invent percentages not implied by the data.
+4. Use simple academic English.
+"""
+    narrative = _call_llm_text(prompt, max_tokens=900)
+    if not narrative:
+        narrative = (
+            f"This draft_2 BERTopic pipeline analyzed {stats['total_papers']} CHB papers across {stats['year_range']}. "
+            "The workflow used SPECTER2 embeddings, UMAP reduction, HDBSCAN clustering, BERTopic topic representation, "
+            "and PAJAIS taxonomy mapping. The resulting abstract-side and title-side themes are saved in the output folder "
+            "for direct inspection and later paper writing."
+        )
+    (output_dir / "narrative.txt").write_text(narrative, encoding="utf-8")
+    meta = {
+        "word_count": len(narrative.split()),
+        "timestamp": pd.Timestamp.utcnow().isoformat(),
+    }
+    _write_json(output_dir / "narrative_meta.json", meta)
+    return {"word_count": meta["word_count"], "preview": narrative[:500]}
+
+
+def run_bertopic_discovery(text_type: str, output_dir: Path = OUTPUT_DIR) -> dict[str, Any]:
+    text_type = text_type.lower()
+    if text_type not in TEXT_TYPE_SETTINGS:
+        raise ValueError("text_type must be 'abstract' or 'title'.")
+
+    output_dir = _ensure_output_dir(output_dir)
+    papers = _load_clean_papers(output_dir)
+    settings = TEXT_TYPE_SETTINGS[text_type]
+
+    analysis_docs = papers["Abstract_Clean"].tolist() if text_type == "abstract" else papers["Title_Clean"].tolist()
+    embedder = Specter2Embedder(batch_size=8)
+    sep_token = AutoTokenizer.from_pretrained(SPECTER2_BASE_MODEL).sep_token
+    embedding_texts = _prepare_embedding_text(papers, text_type, sep_token)
+    embeddings = embedder.encode(embedding_texts, cache_path=output_dir / f"embeddings_{text_type}.npy")
+
+    topic_model = _build_topic_model(text_type)
+    topics, _ = topic_model.fit_transform(analysis_docs, embeddings)
+
+    noise_rate = float(sum(topic_id == -1 for topic_id in topics) / len(topics))
+    discovered_topics = len(set(topic_id for topic_id in topics if topic_id != -1))
+    if noise_rate > 0.10 and discovered_topics >= 10:
+        reduced_topics = topic_model.reduce_outliers(
+            analysis_docs,
+            topics,
+            embeddings=embeddings,
+            strategy="embeddings",
+        )
+        if reduced_topics != topics:
+            topics = reduced_topics
+            topic_model.update_topics(
+                analysis_docs,
+                topics=topics,
+                vectorizer_model=topic_model.vectorizer_model,
+                ctfidf_model=topic_model.ctfidf_model,
+            )
+
+    topic_info = topic_model.get_topic_info()
+    representative_docs = _representative_docs_map(topic_model)
+
+    assignment_df = pd.DataFrame(
+        {
+            "paper_id": papers["Sr No"],
+            "year": papers["Year"],
+            "cited_by": papers["Cited by"],
+            "title": papers["Title_Clean"],
+            "text": analysis_docs,
+            "topic": topics,
+        }
+    )
+    assignment_df["topic_size"] = assignment_df["topic"].map(topic_info.set_index("Topic")["Count"]).fillna(0).astype(int)
+    assignment_df.to_csv(output_dir / f"topic_assignments_{text_type}.csv", index=False, encoding="utf-8-sig")
+
+    topic_rows = []
+    for _, row in topic_info.iterrows():
+        topic_id = int(row["Topic"])
+        topic_rows.append(
+            {
+                "topic_id": topic_id,
+                "count": int(row["Count"]),
+                "name": row.get("Name", ""),
+                "keywords": ", ".join(_topic_keywords(topic_model, topic_id)) if topic_id != -1 else "",
+                "representative_doc_1": representative_docs.get(topic_id, [""])[0] if representative_docs.get(topic_id) else "",
+            }
+        )
+    pd.DataFrame(topic_rows).to_csv(output_dir / f"topic_info_{text_type}.csv", index=False, encoding="utf-8-sig")
+
+    topic_model.save(output_dir / f"bertopic_model_{text_type}", serialization="safetensors", save_ctfidf=True)
+
+    silhouette = _compute_silhouette(embeddings, topics)
+    quality = {
+        "text_type": text_type,
+        "total_documents": int(len(analysis_docs)),
+        "num_topics_excluding_noise": int(sum(topic_id != -1 for topic_id in topic_info["Topic"])),
+        "noise_documents": int(sum(topic_id == -1 for topic_id in topics)),
+        "noise_rate": round(float(sum(topic_id == -1 for topic_id in topics) / len(topics)), 4),
+        "largest_topic_size": int(topic_info[topic_info["Topic"] != -1]["Count"].max()) if (topic_info["Topic"] != -1).any() else 0,
+        "topic_diversity": round(_compute_topic_diversity(topic_model), 4),
+        "silhouette_sample": round(silhouette, 4) if silhouette is not None else None,
+        "umap_n_neighbors": settings["umap_n_neighbors"],
+        "hdbscan_min_cluster_size": settings["hdbscan_min_cluster_size"],
+        "timestamp": pd.Timestamp.utcnow().isoformat(),
+    }
+    _write_json(output_dir / f"quality_{text_type}.json", quality)
+
+    summary_topics = topic_info[topic_info["Topic"] != -1].head(10)
+    summaries = {}
+    for _, row in summary_topics.iterrows():
+        topic_id = int(row["Topic"])
+        summaries[str(topic_id)] = {
+            "size": int(row["Count"]),
+            "keywords": _topic_keywords(topic_model, topic_id),
+            "sample": (representative_docs.get(topic_id, [""])[0] or "")[:200],
+        }
+
+    payload = {
+        "column": "Abstract" if text_type == "abstract" else "Title",
+        "text_type": text_type,
+        "total_texts": int(len(analysis_docs)),
+        "num_topics": int(sum(topic_info["Topic"] != -1)),
+        "top_10_topics": summaries,
+        "timestamp": pd.Timestamp.utcnow().isoformat(),
+    }
+    _write_json(output_dir / f"summaries_{text_type}.json", payload)
+    return payload
+
+
+def label_topics_with_llm(text_type: str, output_dir: Path = OUTPUT_DIR) -> dict[str, Any]:
+    output_dir = _ensure_output_dir(output_dir)
+    topic_model = BERTopic.load(output_dir / f"bertopic_model_{text_type}")
+    payload = _relabel_topics_with_fallback(topic_model, text_type, output_dir)
+    return {
+        "num_labeled": payload["num_labeled"],
+        "sample_labels": {
+            topic_id: info["label"]
+            for topic_id, info in list(payload["topics"].items())[:10]
+        },
+    }
+
+
+def run_full_pipeline(file_path: str | None = None, output_dir: Path = OUTPUT_DIR) -> dict[str, Any]:
+    output_dir = _ensure_output_dir(output_dir)
+    input_path = file_path or str(DEFAULT_INPUT_CSV)
+    stats = load_scopus_csv(input_path, output_dir=output_dir)
+    abstract = run_bertopic_discovery("abstract", output_dir=output_dir)
+    title = run_bertopic_discovery("title", output_dir=output_dir)
+    abstract_labels = label_topics_with_llm("abstract", output_dir=output_dir)
+    title_labels = label_topics_with_llm("title", output_dir=output_dir)
+    abstract_themes = consolidate_into_themes("abstract", TEXT_TYPE_SETTINGS["abstract"]["target_themes"], output_dir)
+    title_themes = consolidate_into_themes("title", TEXT_TYPE_SETTINGS["title"]["target_themes"], output_dir)
+    abstract_taxonomy = compare_with_taxonomy("abstract", output_dir=output_dir)
+    title_taxonomy = compare_with_taxonomy("title", output_dir=output_dir)
+    comparison = generate_comparison_csv(output_dir=output_dir)
+    narrative = export_narrative(output_dir=output_dir)
+
+    manifest = {
+        "input_file": str(input_path),
+        "output_dir": str(output_dir),
+        "method": {
+            "topic_model": "BERTopic",
+            "embedding_model": "SPECTER2",
+            "embedding_input_title_abstract_for_abstract_run": True,
+            "embedding_input_title_only_for_title_run": True,
+            "dimension_reduction": "UMAP",
+            "clustering": "HDBSCAN",
+            "llm_backend": OLLAMA_MODEL,
+        },
+        "stats": stats,
+        "abstract": abstract,
+        "title": title,
+        "abstract_labels": abstract_labels,
+        "title_labels": title_labels,
+        "abstract_themes": abstract_themes,
+        "title_themes": title_themes,
+        "abstract_taxonomy": abstract_taxonomy,
+        "title_taxonomy": title_taxonomy,
+        "comparison": comparison,
+        "narrative": narrative,
+        "timestamp": pd.Timestamp.utcnow().isoformat(),
+    }
+    _write_json(output_dir / "run_manifest.json", manifest)
+    return manifest