Update app.py

app.py

@@ -30,14 +30,13 @@ from typing import Optional, Tuple, Dict, Any
 from agent import PAJAISResearchAgent, AnalysisConfig
 from tools import (
     load_journal_csv, validate_dataframe,
-    PAJAIS_THEMES, export_all_artifacts
-)
-from tools_additions import (
-    dbscan_cluster_topics,
-    enforce_min_membership,
-    split_large_clusters,
+    PAJAIS_THEMES, export_all_artifacts,
+    # Unified clustering pipeline (all now in tools.py)
+    build_title_abstract_column,
+    embed_with_specter2,
+    specter2_hdbscan_cluster_topics,
     get_cluster_summary,
-    label_clusters_with_llm,
+    label_clusters_3llm,
     run_agentic_council,
 )
 
@@ -57,6 +56,7 @@ OUTPUTS_DIR.mkdir(exist_ok=True)
 # ---------------------------------------------------------------------------
 MISTRAL_API_KEY = os.environ.get("MISTRAL_API_KEY", "")
 GEMINI_API_KEY  = os.environ.get("GEMINI_API_KEY", "")
+DEEPSEEK_API_KEY = os.environ.get("DEEPSEEK_API_KEY", "")
 
 # ---------------------------------------------------------------------------
 # Custom CSS — Light, readable theme that works on HuggingFace Spaces
@@ -1461,40 +1461,57 @@ with gr.Blocks(
     # ==================================================================
     # TAB A — DBSCAN Clusters  (Phase 2.5)
     # ==================================================================
-    with gr.Tab("🔵 DBSCAN Clusters"):
-        gr.Markdown("## Phase 2.5: Semantic Clustering via DBSCAN")
+    with gr.Tab("🔵 SPECTER2 Clusters"):
+        gr.Markdown("## Phase 2.5: Semantic Clustering via SPECTER2 → UMAP → HDBSCAN")
         gr.Markdown(
-            "Papers are embedded separately as **title vectors** and **abstract vectors** "
-            "(TF-IDF → LSA), clustered independently with DBSCAN, then merged via weighted vote. "
-            "Large clusters are recursively split; tiny clusters are reassigned or marked noise."
+            "Each paper is represented by **one 768-dim SPECTER2 vector** computed from its "
+            "combined Title + Abstract column (DOI-keyed). "
+            "UMAP reduces dimensions (cosine metric, 50D), then HDBSCAN clusters with an "
+            "automatic parameter sweep to land in the **15–30 cluster** target range. "
+            "Clusters with fewer than 5 or more than 100 papers are automatically merged/split. "
+            "Intra-cluster cosine similarity is kept in the **0.50–0.60** band. "
+            "The 3 most representative paper titles per cluster are sent to "
+            "**Mistral + Gemini + HuggingFace** (all free) for labeling — majority vote wins."
         )
 
         with gr.Accordion("⚙️ Clustering Parameters", open=False):
             with gr.Row():
-                eps_title_slider = gr.Slider(
-                    0.05, 0.60, value=0.25, step=0.01, label="ε Title (cosine distance threshold)"
+                min_cs_slider = gr.Slider(
+                    2, 20, value=5, step=1,
+                    label="Min Cluster Size (papers)",
+                    info="Papers < this → merged into nearest cluster"
                 )
-                eps_abstract_slider = gr.Slider(
-                    0.05, 0.60, value=0.30, step=0.01, label="ε Abstract"
+                max_cs_slider = gr.Slider(
+                    20, 200, value=100, step=5,
+                    label="Max Cluster Size (papers)",
+                    info="Papers > this → cluster is split"
                 )
             with gr.Row():
-                min_samples_slider = gr.Slider(
-                    2, 20, value=2, step=1, label="Min Samples (DBSCAN core-point threshold)"
+                target_min_slider = gr.Slider(
+                    5, 20, value=15, step=1,
+                    label="Target Min Clusters",
+                    info="HDBSCAN sweep lower bound"
                 )
-                min_members_slider = gr.Slider(
-                    2, 20, value=3, step=1, label="Min Cluster Membership (post-processing)"
+                target_max_slider = gr.Slider(
+                    15, 40, value=30, step=1,
+                    label="Target Max Clusters",
+                    info="HDBSCAN sweep upper bound"
                 )
             with gr.Row():
-                max_size_slider = gr.Slider(
-                    10, 200, value=30, step=5, label="Max Cluster Size (triggers splitting)"
+                sim_low_slider = gr.Slider(
+                    0.30, 0.70, value=0.50, step=0.01,
+                    label="Min Cosine Similarity (cluster quality)",
+                    info="Clusters below this are dissolved to noise"
                 )
-                vote_weight_slider = gr.Slider(
-                    0.0, 1.0, value=0.6, step=0.05, label="Abstract Vote Weight (vs Title)"
+                umap_neighbors_slider = gr.Slider(
+                    5, 50, value=15, step=1,
+                    label="UMAP n_neighbors",
+                    info="Controls local vs global structure"
                 )
 
         with gr.Row():
-            btn_run_dbscan = gr.Button("▶ Run DBSCAN Clustering", variant="primary")
-            btn_llm_label = gr.Button("🤖 Label Clusters with LLM", variant="secondary")
+            btn_run_dbscan = gr.Button("▶ Run SPECTER2 → UMAP → HDBSCAN", variant="primary")
+            btn_llm_label = gr.Button("🤖 Label Clusters (3 LLMs)", variant="secondary")
 
         dbscan_status = gr.Markdown("*Run DBSCAN or use the full pipeline from Tab 1.*")
 
@@ -1527,7 +1544,7 @@ with gr.Blocks(
 
         def handle_run_dbscan(
             df, existing_cluster_df, existing_summary,
-            eps_t, eps_a, min_s, min_m, max_sz, vote_w,
+            min_cs, max_cs, target_min, target_max, sim_low, umap_n,
             progress=gr.Progress(track_tqdm=True)
         ):
             if existing_cluster_df is not None and not existing_cluster_df.empty:
@@ -1536,7 +1553,7 @@ with gr.Blocks(
                 saved_docs = _safe_save_csv(existing_cluster_df, "cluster_documents.csv")
                 saved_sum = _safe_save_csv(summary, "cluster_summary.csv")
                 return (
-                    "<div class='success-box'>✅ Loaded existing DBSCAN results.</div>",
+                    "<div class='success-box'>✅ Loaded existing results.</div>",
                     summary, existing_cluster_df, summary,
                     fig_sz, fig_noise,
                     gr.update(value=saved_docs), gr.update(value=saved_sum), gr.update(),
@@ -1552,19 +1569,26 @@ with gr.Blocks(
 
             try:
                 _ensure_output_dir()
-                progress(0.1, desc="Vectorising documents…")
-                cdf = dbscan_cluster_topics(
-                    df,
-                    eps_title=eps_t, eps_abstract=eps_a,
-                    min_samples=int(min_s),
-                    n_svd_components=64,
-                    vote_weight_abstract=vote_w,
+                progress(0.05, desc="Building title+abstract column…")
+                df_ta = build_title_abstract_column(df)
+
+                progress(0.15, desc="Generating SPECTER2 embeddings (may take 2-5 min)…")
+                texts = df_ta['title_abstract'].tolist()
+                embs = embed_with_specter2(texts, cache_dir='outputs/specter_cache')
+
+                progress(0.60, desc="UMAP + HDBSCAN clustering…")
+                cdf = specter2_hdbscan_cluster_topics(
+                    df=df_ta,
+                    embeddings=embs,
+                    min_cluster_size=int(min_cs),
+                    max_cluster_size=int(max_cs),
+                    target_min_clusters=int(target_min),
+                    target_max_clusters=int(target_max),
+                    cosine_sim_low=float(sim_low),
+                    cosine_sim_high=float(sim_low) + 0.10,
+                    umap_n_neighbors=int(umap_n),
                 )
-                progress(0.5, desc="Enforcing min membership…")
-                cdf = enforce_min_membership(cdf, min_members=int(min_m))
-                progress(0.7, desc="Splitting large clusters…")
-                cdf = split_large_clusters(cdf, max_cluster_size=int(max_sz))
-                progress(0.9, desc="Summarising…")
+                progress(0.85, desc="Summarising clusters…")
                 summary = get_cluster_summary(cdf)
                 progress(1.0, desc="Done!")
 
@@ -1572,8 +1596,8 @@ with gr.Blocks(
                 saved_docs = _safe_save_csv(cdf, "cluster_documents.csv")
                 saved_sum = _safe_save_csv(summary, "cluster_summary.csv")
 
-                n_c = len(set(cdf["cluster_final"]) - {-1})
-                n_n = int(cdf["is_noise"].sum())
+                n_c = len(set(cdf['cluster_final']) - {-1})
+                n_n = int(cdf['is_noise'].sum())
                 return (
                     f"<div class='success-box'>✅ {n_c} clusters found, {n_n} noise docs.</div>",
                     summary, cdf, summary,
@@ -1591,21 +1615,36 @@ with gr.Blocks(
         def handle_llm_label(cluster_df, cluster_summary, progress=gr.Progress(track_tqdm=True)):
             if cluster_df is None or cluster_df.empty:
                 return (
-                    "<div class='error-box'>❌ Run DBSCAN first.</div>",
+                    "<div class='error-box'>❌ Run clustering first.</div>",
                     cluster_summary, gr.update()
                 )
             try:
                 _ensure_output_dir()
-                progress(0.2, desc="Sending clusters to LLM…")
-                labeled = label_clusters_with_llm(
+                # Load cached embeddings if available
+                import glob
+                cache_files = glob.glob('outputs/specter_cache/*.npy')
+                if not cache_files:
+                    return (
+                        "<div class='error-box'>❌ No SPECTER2 cache found. Run clustering tab first.</div>",
+                        cluster_summary, gr.update()
+                    )
+                embs = np.load(sorted(cache_files)[-1])  # most recent cache
+
+                progress(0.2, desc="Sending clusters to LLMs…")
+                labeled = label_clusters_3llm(
                     cluster_df=cluster_df,
-                    cluster_summary_df=cluster_summary.copy() if cluster_summary is not None else get_cluster_summary(cluster_df),
-                    max_clusters=50,
+                    cluster_summary_df=cluster_summary.copy() if cluster_summary is not None
+                        else get_cluster_summary(cluster_df),
+                    embeddings=embs,
+                    mistral_api_key=MISTRAL_API_KEY,
+                    gemini_api_key=GEMINI_API_KEY,
+                    deepseek_api_key=DEEPSEEK_API_KEY,
+                    max_clusters=30,
                 )
                 progress(1.0, desc="Done!")
                 saved = _safe_save_csv(labeled, "cluster_labels.csv")
                 return (
-                    "<div class='success-box'>✅ Clusters labeled by LLM.</div>",
+                    "<div class='success-box'>✅ Clusters labeled by 3 LLMs (majority vote).</div>",
                     labeled,
                     gr.update(value=saved),
                 )
@@ -1619,9 +1658,9 @@ with gr.Blocks(
             fn=handle_run_dbscan,
             inputs=[
                 state_df, state_cluster_df, state_cluster_summary,
-                eps_title_slider, eps_abstract_slider,
-                min_samples_slider, min_members_slider,
-                max_size_slider, vote_weight_slider,
+                min_cs_slider, max_cs_slider,
+                target_min_slider, target_max_slider,
+                sim_low_slider, umap_neighbors_slider,
             ],
             outputs=[
                 dbscan_status,
@@ -1653,9 +1692,10 @@ with gr.Blocks(
     with gr.Tab("🧠 Agentic Council"):
         gr.Markdown("## Phase 6.5: Dual-Model Research Council")
         gr.Markdown(
-            "Two AI models independently assess the PAJAIS research gap findings:\n"
+            "Three AI models independently assess the PAJAIS research gap findings:\n"
             "- **Mistral** (Panel A) — pragmatic applied IS perspective\n"
-            "- **Gemini** (Panel B) — broad technology futures perspective\n\n"
+            "- **Gemini** (Panel B) — broad technology futures perspective\n"
+            "- **DeepSeek** (Panel C) — deep analytical synthesis\n\n"
             "API keys are loaded automatically from HuggingFace Secrets "
             "(`MISTRAL_API_KEY`, `GEMINI_API_KEY`). "
             "Configure them in your Space under **Settings → Variables and Secrets**."
@@ -1664,8 +1704,9 @@ with gr.Blocks(
         # Key-status indicator — shows which secrets are present at load time
         _key_lines = ["**🔑 Secret Status (loaded at startup):**"]
         for _label, _val in [
-            ("MISTRAL_API_KEY", MISTRAL_API_KEY),
-            ("GEMINI_API_KEY",  GEMINI_API_KEY),
+            ("MISTRAL_API_KEY",  MISTRAL_API_KEY),
+            ("GEMINI_API_KEY",   GEMINI_API_KEY),
+            ("DEEPSEEK_API_KEY", DEEPSEEK_API_KEY),
         ]:
             _icon = "✅ present" if _val else "❌ missing"
             _key_lines.append(f"- `{_label}`: {_icon}")
@@ -1691,6 +1732,14 @@ with gr.Blocks(
                     interactive=False,
                     show_label=True,
                 )
+            with gr.Column():
+                gr.Markdown("### 🟣 Panel C — DeepSeek")
+                deepseek_output = gr.Textbox(
+                    label="DeepSeek Assessment",
+                    lines=18,
+                    interactive=False,
+                    show_label=True,
+                )
 
         dl_council = gr.DownloadButton("⬇ council_report.json", value=None)
 
@@ -1721,6 +1770,7 @@ with gr.Blocks(
                     topic_df=topic_df,
                     mistral_api_key=MISTRAL_API_KEY,
                     gemini_api_key=GEMINI_API_KEY,
+                    deepseek_api_key=DEEPSEEK_API_KEY,
                 )
                 progress(0.9, desc="Saving report…")
                 saved = _safe_save_json(result, "council_report.json")
@@ -1731,18 +1781,19 @@ with gr.Blocks(
                     status,
                     result.get("mistral", ""),
                     result.get("gemini", ""),
+                    result.get("deepseek", ""),
                     gr.update(value=saved),
                 )
             except Exception as e:
                 return (
                     f"<div class='error-box'>❌ Council failed: {e}</div>",
-                    "", "", gr.update()
+                    "", "", "", gr.update()
                 )
 
         btn_run_council.click(
             fn=handle_run_council,
             inputs=[state_taxonomy_map, state_topic_df],
-            outputs=[council_status, mistral_output, gemini_output, dl_council]
+            outputs=[council_status, mistral_output, gemini_output, deepseek_output, dl_council]
         )
 
         # Auto-fill if council already ran (e.g. via full pipeline)
@@ -1750,9 +1801,10 @@ with gr.Blocks(
             fn=lambda cr: (
                 cr.get("mistral", "") if cr else "",
                 cr.get("gemini", "") if cr else "",
+                cr.get("deepseek", "") if cr else "",
             ),
             inputs=[state_council_result],
-            outputs=[mistral_output, gemini_output]
+            outputs=[mistral_output, gemini_output, deepseek_output]
         )
 
     # ==================================================================