Update agent.py

agent.py

@@ -27,25 +27,16 @@ from tools import (
     generate_section7_narrative,
     export_all_artifacts,
     PAJAIS_THEMES,
+    # New unified pipeline (Groups 0, 8-11 in tools.py)
+    build_title_abstract_column,
+    embed_with_specter2,
+    specter2_hdbscan_cluster_topics,
+    get_cluster_summary,
+    label_clusters_3llm,
+    run_agentic_council,
 )
 
-# NEW ADDITIONS — import from tools_additions (or inline if merged into tools.py)
-try:
-    from tools_additions import (
-        dbscan_cluster_topics,
-        enforce_min_membership,
-        split_large_clusters,
-        get_cluster_summary,
-        label_clusters_with_llm,
-        run_agentic_council,
-    )
-    _ADDITIONS_AVAILABLE = True
-except ImportError:
-    _ADDITIONS_AVAILABLE = False
-    import logging as _log
-    _log.getLogger(__name__).warning(
-        "tools_additions.py not found — DBSCAN and Council features disabled."
-    )
+_ADDITIONS_AVAILABLE = True  # everything is now in tools.py
 
 # ---------------------------------------------------------------------------
 # Logging setup
@@ -99,25 +90,25 @@ class AnalysisConfig:
     publishable_min_docs: int = 5
     publishable_min_coherence: float = 0.3
 
-    # DBSCAN clustering
-    dbscan_eps_title: float = 0.25
-    dbscan_eps_abstract: float = 0.30
-    dbscan_min_samples: int = 2
-    dbscan_n_svd: int = 64
-    dbscan_vote_weight_abstract: float = 0.6
-
-    # Cluster post-processing
-    min_cluster_members: int = 3
-    max_cluster_size: int = 30
-    dbscan_eps_split: float = 0.20
-
-    # LLM labeling
-    llm_label_max_clusters: int = 50
-
-    # Agentic council API keys (populated from env or UI)
+    # SPECTER2 + UMAP + HDBSCAN clustering
+    specter2_batch_size: int = 8
+    specter2_cache_dir: str = "outputs/specter_cache"
+    umap_n_components: int = 50
+    umap_n_neighbors: int = 15
+    hdbscan_min_cluster_size: int = 5
+    hdbscan_max_cluster_size: int = 100
+    cluster_target_min: int = 15
+    cluster_target_max: int = 30
+    cosine_sim_low: float = 0.50
+    cosine_sim_high: float = 0.60
+
+    # LLM labeling (all free APIs)
+    llm_label_max_clusters: int = 30
+
+    # API keys (populated from env or UI)
     mistral_api_key: str = ""
     gemini_api_key: str = ""
-    anthropic_api_key: str = ""
+    deepseek_api_key: str = ""  # DeepSeek API key (panel C + synthesis judge)
 
 
 # ---------------------------------------------------------------------------
@@ -143,12 +134,13 @@ class PAJAISResearchAgent:
         self.artifacts: Dict[str, str] = {}
         self.supplementary_insights: Dict[str, Any] = {}
 
-        # NEW: DBSCAN state
+        # SPECTER2 + HDBSCAN state
+        self.specter2_embeddings: Optional[np.ndarray] = None   # (N, 768)
         self.cluster_df: Optional[pd.DataFrame] = None          # doc-level
         self.cluster_summary_df: Optional[pd.DataFrame] = None  # cluster-level
         self.cluster_labeled_df: Optional[pd.DataFrame] = None  # with LLM labels
 
-        # NEW: Agentic council
+        # Agentic council
         self.council_result: Optional[Dict[str, str]] = None
 
         self._errors: List[str] = []
@@ -424,77 +416,73 @@ class PAJAISResearchAgent:
     # -----------------------------------------------------------------------
 
     def _phase2_5_dbscan_clustering(self) -> None:
-        """Phase 2.5: Cluster papers via DBSCAN on title + abstract vectors."""
-        if not _ADDITIONS_AVAILABLE:
-            logger.warning("Phase 2.5: tools_additions not available; skipping DBSCAN.")
-            return
+        """Phase 2.5: SPECTER2 embeddings → UMAP → HDBSCAN (15-30 clusters)."""
         if self.df is None or self.df.empty:
             raise ValueError("Phase 2.5: No data loaded. Run Phase 1 first.")
 
-        logger.info("Phase 2.5: Running DBSCAN clustering...")
-
-        # Step A: Initial DBSCAN
-        self.cluster_df = dbscan_cluster_topics(
-            df=self.df,
-            eps_title=self.config.dbscan_eps_title,
-            eps_abstract=self.config.dbscan_eps_abstract,
-            min_samples=self.config.dbscan_min_samples,
-            n_svd_components=self.config.dbscan_n_svd,
-            vote_weight_abstract=self.config.dbscan_vote_weight_abstract,
+        logger.info("Phase 2.5: Building title+abstract combined column...")
+        df_ta = build_title_abstract_column(self.df)
+        # Store back so downstream code can access title_abstract and doi_key
+        self.df = df_ta
+
+        logger.info("Phase 2.5: Generating SPECTER2 embeddings (one per paper)...")
+        texts = df_ta['title_abstract'].tolist()
+        self.specter2_embeddings = embed_with_specter2(
+            texts=texts,
+            cache_dir=self.config.specter2_cache_dir,
+            batch_size=self.config.specter2_batch_size,
         )
 
-        # Step B: Min membership enforcement
-        self.cluster_df = enforce_min_membership(
-            self.cluster_df,
-            min_members=self.config.min_cluster_members,
-            strategy="reassign_to_nearest",
+        logger.info("Phase 2.5: Running UMAP + HDBSCAN clustering...")
+        self.cluster_df = specter2_hdbscan_cluster_topics(
+            df=df_ta,
+            embeddings=self.specter2_embeddings,
+            min_cluster_size=self.config.hdbscan_min_cluster_size,
+            max_cluster_size=self.config.hdbscan_max_cluster_size,
+            target_min_clusters=self.config.cluster_target_min,
+            target_max_clusters=self.config.cluster_target_max,
+            cosine_sim_low=self.config.cosine_sim_low,
+            cosine_sim_high=self.config.cosine_sim_high,
+            umap_n_components=self.config.umap_n_components,
+            umap_n_neighbors=self.config.umap_n_neighbors,
+            random_state=self.config.random_state,
         )
 
-        # Step C: Split large clusters
-        self.cluster_df = split_large_clusters(
-            self.cluster_df,
-            max_cluster_size=self.config.max_cluster_size,
-            eps_split=self.config.dbscan_eps_split,
-            min_samples_split=self.config.dbscan_min_samples,
-            max_depth=3,
-        )
-
-        # Step D: Build summary
         self.cluster_summary_df = get_cluster_summary(self.cluster_df)
 
-        n_clusters = len(set(self.cluster_df["cluster_final"]) - {-1})
-        n_noise = int(self.cluster_df["is_noise"].sum())
-        logger.info(
-            f"Phase 2.5 complete: {n_clusters} clusters, {n_noise} noise docs."
-        )
+        n_clusters = len(set(self.cluster_df['cluster_final']) - {-1})
+        n_noise = int(self.cluster_df['is_noise'].sum())
+        logger.info(f"Phase 2.5 complete: {n_clusters} clusters, {n_noise} noise docs.")
 
     def run_llm_cluster_labeling(
         self,
-        api_key: Optional[str] = None,
-        model: str = "claude-sonnet-4-20250514",
+        mistral_key: str = '',
+        gemini_key: str = '',
+        deepseek_key: str = '',
     ) -> Optional[pd.DataFrame]:
-        """
-        Label clusters using the Anthropic LLM.
+        """Label clusters using 3 LLMs: Mistral + Gemini + DeepSeek.
+
+        Majority vote selects the final label; all 3 candidates stored.
         Can be called independently after phase 2.5.
         """
-        if not _ADDITIONS_AVAILABLE:
-            logger.warning("LLM labeling: tools_additions not available.")
-            return None
         if self.cluster_df is None or self.cluster_summary_df is None:
-            logger.warning("LLM labeling: run DBSCAN clustering first.")
+            logger.warning("LLM labeling: run SPECTER2/HDBSCAN clustering first.")
+            return None
+        if self.specter2_embeddings is None:
+            logger.warning("LLM labeling: specter2_embeddings not available.")
             return None
 
-        key = api_key or self.config.anthropic_api_key
-        self.cluster_labeled_df = label_clusters_with_llm(
+        self.cluster_labeled_df = label_clusters_3llm(
             cluster_df=self.cluster_df,
             cluster_summary_df=self.cluster_summary_df.copy(),
-            api_key=key,
-            model=model,
+            embeddings=self.specter2_embeddings,
+            mistral_api_key=mistral_key or self.config.mistral_api_key,
+            gemini_api_key=gemini_key or self.config.gemini_api_key,
+            deepseek_api_key=deepseek_key or self.config.deepseek_api_key,
             max_clusters=self.config.llm_label_max_clusters,
         )
 
-        # Persist
-        out = Path(self.config.output_dir) / "cluster_labels.csv"
+        out = Path(self.config.output_dir) / 'cluster_labels.csv'
         try:
             self.cluster_labeled_df.to_csv(out, index=False)
             logger.info(f"Saved cluster_labels.csv ({len(self.cluster_labeled_df)} rows)")
@@ -594,7 +582,7 @@ class PAJAISResearchAgent:
             topic_df=self.topic_df,
             mistral_api_key=self.config.mistral_api_key,
             gemini_api_key=self.config.gemini_api_key,
-            anthropic_api_key=self.config.anthropic_api_key,
+            deepseek_api_key=self.config.deepseek_api_key,
         )
 
         # Persist council report