# ============================================================================= # agent.py -- PAJAIS Research Intelligence Agent (v2 — with DBSCAN + Council) # Deterministic six-phase orchestration pipeline + Phase 2.5 (DBSCAN) + # Phase 6.5 (Agentic Council) # ============================================================================= import logging import logging.handlers import json from dataclasses import dataclass, field from pathlib import Path from typing import Any, Callable, Dict, List, Optional import numpy as np import pandas as pd from tools import ( load_journal_csv, validate_dataframe, preprocess_corpus, run_lda_topic_model, build_topic_dataframe, auto_label_topic, map_topics_to_pajais, generate_taxonomy_map, compare_abstract_vs_title_themes, generate_section7_narrative, export_all_artifacts, PAJAIS_THEMES, # New unified pipeline (Groups 0, 8-11 in tools.py) build_title_abstract_column, embed_with_specter2, specter2_hdbscan_cluster_topics, get_cluster_summary, label_clusters_3llm, run_agentic_council, ) _ADDITIONS_AVAILABLE = True # everything is now in tools.py # --------------------------------------------------------------------------- # Logging setup # --------------------------------------------------------------------------- def _setup_logger() -> logging.Logger: """Configure module logger to file and console.""" log = logging.getLogger('PAJAISAgent') if log.handlers: return log log.setLevel(logging.DEBUG) fmt = logging.Formatter( '%(asctime)s - PAJAISAgent - %(levelname)s - %(message)s' ) ch = logging.StreamHandler() ch.setLevel(logging.INFO) ch.setFormatter(fmt) log.addHandler(ch) Path('outputs').mkdir(exist_ok=True) try: fh = logging.FileHandler('outputs/agent.log', mode='a', encoding='utf-8') fh.setLevel(logging.DEBUG) fh.setFormatter(fmt) log.addHandler(fh) except OSError: pass return log logger = _setup_logger() # --------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------- @dataclass class AnalysisConfig: """Configuration for the PAJAIS analysis pipeline.""" # LDA n_topics: int = 40 min_topics_required: int = 98 n_lda_passes: int = 15 random_state: int = 42 output_dir: str = "outputs" # Taxonomy pajais_match_threshold: float = 0.15 publishable_min_docs: int = 5 publishable_min_coherence: float = 0.3 # SPECTER2 + UMAP + HDBSCAN clustering specter2_batch_size: int = 8 specter2_cache_dir: str = "outputs/specter_cache" umap_n_components: int = 50 umap_n_neighbors: int = 15 hdbscan_min_cluster_size: int = 5 hdbscan_max_cluster_size: int = 100 cluster_target_min: int = 15 cluster_target_max: int = 30 cosine_sim_low: float = 0.50 cosine_sim_high: float = 0.60 # LLM labeling (all free APIs) llm_label_max_clusters: int = 30 # API keys (populated from env or UI) mistral_api_key: str = "" gemini_api_key: str = "" ollama_url: str = "http://localhost:11434" # Local Ollama URL # --------------------------------------------------------------------------- # Agent class # --------------------------------------------------------------------------- class PAJAISResearchAgent: """Deterministic research gap analysis pipeline + DBSCAN + Agentic Council.""" def __init__(self, config: Optional[AnalysisConfig] = None) -> None: self.config = config or AnalysisConfig() Path(self.config.output_dir).mkdir(parents=True, exist_ok=True) # Core state (original pipeline) self.df: Optional[pd.DataFrame] = None self.validation: Optional[Dict[str, Any]] = None self.processed_texts: Optional[List[str]] = None self.lda_result: Optional[Dict[str, Any]] = None self.topic_df: Optional[pd.DataFrame] = None self.comparison_df: Optional[pd.DataFrame] = None self.taxonomy_map: Optional[Dict[str, Any]] = None self.narrative: str = '' self.artifacts: Dict[str, str] = {} self.supplementary_insights: Dict[str, Any] = {} # SPECTER2 + HDBSCAN state self.specter2_embeddings: Optional[np.ndarray] = None # (N, 768) self.cluster_df: Optional[pd.DataFrame] = None # doc-level self.cluster_summary_df: Optional[pd.DataFrame] = None # cluster-level self.cluster_labeled_df: Optional[pd.DataFrame] = None # with LLM labels # Agentic council self.council_result: Optional[Dict[str, str]] = None self._errors: List[str] = [] self._warnings: List[str] = [] self._phases_completed: List[int] = [] # ----------------------------------------------------------------------- # Public pipeline entry point # ----------------------------------------------------------------------- def run_full_pipeline( self, file_path: str, on_progress: Optional[Callable[[int, str, float], None]] = None, run_council: bool = False, ) -> Dict[str, Any]: """Execute all phases sequentially.""" self._errors = [] self._warnings = [] self._phases_completed = [] def _progress(phase: int, msg: str, pct: float) -> None: logger.info(f"[Phase {phase}] {msg} ({pct:.0f}%)") if on_progress: try: on_progress(phase, msg, pct) except Exception as cb_err: logger.warning(f"Progress callback error: {cb_err}") # Phase 1 — Data Ingestion _progress(1, "Loading and validating data...", 0.0) try: self._phase1_data_ingestion(file_path) self._phases_completed.append(1) _progress(1, "Data loaded.", 12.0) except Exception as e: self._errors.append(f"Phase 1 failed: {e}") logger.error(f"Phase 1 error: {e}", exc_info=True) # Phase 2 — LDA Topic Modeling _progress(2, "Running LDA topic modeling...", 12.0) try: self._phase2_topic_modeling(on_progress=on_progress) self._phases_completed.append(2) _progress(2, "Topic modeling complete.", 28.0) except Exception as e: self._errors.append(f"Phase 2 failed: {e}") logger.error(f"Phase 2 error: {e}", exc_info=True) # Phase 2.5 — DBSCAN Clustering (NEW) _progress(2, "Running DBSCAN clustering...", 28.0) try: self._phase2_5_dbscan_clustering() _progress(2, "DBSCAN clustering complete.", 38.0) except Exception as e: self._errors.append(f"Phase 2.5 failed: {e}") logger.error(f"Phase 2.5 error: {e}", exc_info=True) # Phase 3 — Export Topic Table _progress(3, "Exporting topic review table...", 38.0) try: self._phase3_export_topic_table() self._phases_completed.append(3) _progress(3, "Topic table exported.", 48.0) except Exception as e: self._errors.append(f"Phase 3 failed: {e}") logger.error(f"Phase 3 error: {e}", exc_info=True) # Phase 4 — Abstract vs Title Comparison _progress(4, "Comparing abstracts vs titles...", 48.0) try: self._phase4_abstract_title_comparison() self._phases_completed.append(4) _progress(4, "Abstract/title comparison done.", 60.0) except Exception as e: self._errors.append(f"Phase 4 failed: {e}") logger.error(f"Phase 4 error: {e}", exc_info=True) # Phase 5 — PAJAIS Taxonomy Mapping _progress(5, "Mapping to PAJAIS taxonomy...", 60.0) try: self._phase5_taxonomy_mapping() self._phases_completed.append(5) _progress(5, "Taxonomy mapping complete.", 72.0) except Exception as e: self._errors.append(f"Phase 5 failed: {e}") logger.error(f"Phase 5 error: {e}", exc_info=True) # Phase 5.5 — Mapping display try: self._phase5_5_mapping_display() _progress(5, "Mapping display saved.", 75.0) except Exception as e: self._errors.append(f"Phase 5.5 failed: {e}") logger.error(f"Phase 5.5 error: {e}", exc_info=True) # Phase 6 — Narrative _progress(6, "Generating narrative...", 75.0) try: self._phase6_narrative() self._phases_completed.append(6) _progress(6, "Narrative generated.", 85.0) except Exception as e: self._errors.append(f"Phase 6 failed: {e}") logger.error(f"Phase 6 error: {e}", exc_info=True) # Phase 6.5 — Agentic Council (optional; requires API keys) if run_council and _ADDITIONS_AVAILABLE: _progress(6, "Running Agentic Council...", 85.0) try: self._phase6_5_agentic_council() _progress(6, "Council complete.", 93.0) except Exception as e: self._errors.append(f"Phase 6.5 failed: {e}") logger.error(f"Phase 6.5 error: {e}", exc_info=True) # Export artifacts try: self.artifacts = export_all_artifacts( topic_df=self.topic_df if self.topic_df is not None else pd.DataFrame(), taxonomy_map=self.taxonomy_map or {}, comparison_df=self.comparison_df if self.comparison_df is not None else pd.DataFrame(), narrative=self.narrative, output_dir=self.config.output_dir, ) self._export_dbscan_artifacts() except Exception as e: self._errors.append(f"Artifact export failed: {e}") # Supplementary analytics try: self._discover_supplementary_insights() except Exception as e: logger.warning(f"Supplementary insights failed: {e}") _progress(6, "Pipeline complete.", 100.0) return self._build_summary() # ----------------------------------------------------------------------- # run_phase — single phase execution # ----------------------------------------------------------------------- def run_phase(self, phase_num: int, **kwargs) -> Dict[str, Any]: phase_map = { 1: lambda: self._phase1_data_ingestion(kwargs.get('file_path', '')), 2: lambda: self._phase2_topic_modeling(), 25: lambda: self._phase2_5_dbscan_clustering(), 3: lambda: self._phase3_export_topic_table(), 4: lambda: self._phase4_abstract_title_comparison(), 5: lambda: self._phase5_taxonomy_mapping(), 6: lambda: self._phase6_narrative(), 65: lambda: self._phase6_5_agentic_council(), } if phase_num not in phase_map: return {'success': False, 'error': f'Unknown phase: {phase_num}'} try: phase_map[phase_num]() if phase_num not in self._phases_completed: self._phases_completed.append(phase_num) return {'success': True, 'phase': phase_num} except Exception as e: logger.error(f"run_phase({phase_num}) failed: {e}", exc_info=True) return {'success': False, 'phase': phase_num, 'error': str(e)} # ----------------------------------------------------------------------- # Phase implementations — original # ----------------------------------------------------------------------- def _phase1_data_ingestion(self, file_path: str) -> None: logger.info(f"Phase 1: Loading {file_path}") self.df = load_journal_csv(file_path) self.validation = validate_dataframe(self.df) if self.validation: for w in self.validation.get('warnings', []): self._warnings.append(w) logger.warning(f"Validation warning: {w}") row_count = self.validation.get('row_count', 0) if row_count < 50: logger.warning(f"Small dataset ({row_count} rows). Continuing.") logger.info(f"Phase 1 complete: {len(self.df)} rows loaded.") def _phase2_topic_modeling( self, on_progress: Optional[Callable] = None ) -> None: if self.df is None or self.df.empty: raise ValueError("Phase 2: No data loaded. Run Phase 1 first.") abstracts = self.df.get('abstract', pd.Series(dtype=str)).fillna('').tolist() non_empty_abstracts = [t for t in abstracts if isinstance(t, str) and t.strip()] if len(non_empty_abstracts) < 5: logger.warning("Abstracts mostly empty; falling back to titles.") titles = self.df.get('title', pd.Series(dtype=str)).fillna('').tolist() texts_to_process = [t for t in titles if isinstance(t, str) and t.strip()] else: texts_to_process = non_empty_abstracts logger.info(f"Phase 2: Preprocessing {len(texts_to_process)} texts...") if on_progress: on_progress(2, "Preprocessing texts...", 14.0) self.processed_texts = preprocess_corpus(texts_to_process, n_jobs=1) non_empty_processed = [t for t in self.processed_texts if t.strip()] if on_progress: on_progress(2, "Running LDA topic modeling...", 20.0) logger.info(f"Phase 2: Running LDA with n_topics={self.config.n_topics}") try: self.lda_result = run_lda_topic_model( texts=non_empty_processed, n_topics=self.config.n_topics, n_passes=self.config.n_lda_passes, random_state=self.config.random_state, ) except Exception as gensim_err: logger.warning(f"Gensim LDA failed ({gensim_err}), falling back to NMF.") self.lda_result = self._fallback_nmf(non_empty_processed) if on_progress: on_progress(2, "Building topic dataframe...", 26.0) self.topic_df = build_topic_dataframe(self.lda_result) logger.info(f"Phase 2 complete: {len(self.topic_df)} topics extracted.") def _fallback_nmf(self, texts: List[str]) -> Dict[str, Any]: logger.info("Attempting NMF fallback topic modeling...") try: from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.decomposition import NMF vectorizer = TfidfVectorizer(max_features=3000, min_df=2) tfidf = vectorizer.fit_transform(texts) feature_names = vectorizer.get_feature_names_out() n_topics = min(self.config.n_topics, max(5, tfidf.shape[0] // 3)) nmf = NMF(n_components=n_topics, random_state=self.config.random_state) nmf.fit(tfidf) topic_words = [] for topic_idx, topic in enumerate(nmf.components_): top_indices = topic.argsort()[:-16:-1] topic_words.append([ (feature_names[i], float(topic[i])) for i in top_indices ]) doc_topic_matrix = nmf.transform(tfidf) doc_topics = [] for row in doc_topic_matrix: doc_topics.append([(i, float(prob)) for i, prob in enumerate(row)]) return { 'model': nmf, 'corpus': tfidf, 'dictionary': None, 'topic_words': topic_words, 'coherence_score': 0.25, 'doc_topics': doc_topics, } except Exception as e: logger.error(f"NMF fallback failed: {e}") return { 'model': None, 'corpus': [], 'dictionary': None, 'topic_words': [], 'coherence_score': 0.0, 'doc_topics': [] } # ----------------------------------------------------------------------- # Phase 2.5 — DBSCAN Clustering (NEW) # ----------------------------------------------------------------------- def _phase2_5_dbscan_clustering(self) -> None: """Phase 2.5: SPECTER2 embeddings → UMAP → HDBSCAN (15-30 clusters).""" if self.df is None or self.df.empty: raise ValueError("Phase 2.5: No data loaded. Run Phase 1 first.") logger.info("Phase 2.5: Building title+abstract combined column...") df_ta = build_title_abstract_column(self.df) # Store back so downstream code can access title_abstract and doi_key self.df = df_ta logger.info("Phase 2.5: Generating SPECTER2 embeddings (one per paper)...") texts = df_ta['title_abstract'].tolist() self.specter2_embeddings = embed_with_specter2( texts=texts, cache_dir=self.config.specter2_cache_dir, batch_size=self.config.specter2_batch_size, ) logger.info("Phase 2.5: Running UMAP + HDBSCAN clustering...") self.cluster_df = specter2_hdbscan_cluster_topics( df=df_ta, embeddings=self.specter2_embeddings, min_cluster_size=self.config.hdbscan_min_cluster_size, max_cluster_size=self.config.hdbscan_max_cluster_size, target_min_clusters=self.config.cluster_target_min, target_max_clusters=self.config.cluster_target_max, cosine_sim_low=self.config.cosine_sim_low, cosine_sim_high=self.config.cosine_sim_high, umap_n_components=self.config.umap_n_components, umap_n_neighbors=self.config.umap_n_neighbors, random_state=self.config.random_state, ) self.cluster_summary_df = get_cluster_summary(self.cluster_df) n_clusters = len(set(self.cluster_df['cluster_final']) - {-1}) n_noise = int(self.cluster_df['is_noise'].sum()) logger.info(f"Phase 2.5 complete: {n_clusters} clusters, {n_noise} noise docs.") def run_llm_cluster_labeling( self, mistral_key: str = '', gemini_key: str = '', ollama_url: str = '', ) -> Optional[pd.DataFrame]: """Label clusters using 3 LLMs: Mistral + Gemini + Ollama. Majority vote selects the final label; all 3 candidates stored. Can be called independently after phase 2.5. """ if self.cluster_df is None or self.cluster_summary_df is None: logger.warning("LLM labeling: run SPECTER2/HDBSCAN clustering first.") return None if self.specter2_embeddings is None: logger.warning("LLM labeling: specter2_embeddings not available.") return None self.cluster_labeled_df = label_clusters_3llm( cluster_df=self.cluster_df, cluster_summary_df=self.cluster_summary_df.copy(), embeddings=self.specter2_embeddings, mistral_api_key=mistral_key or self.config.mistral_api_key, gemini_api_key=gemini_key or self.config.gemini_api_key, ollama_url=ollama_url or self.config.ollama_url, max_clusters=self.config.llm_label_max_clusters, ) out = Path(self.config.output_dir) / 'cluster_labels.csv' try: self.cluster_labeled_df.to_csv(out, index=False) logger.info(f"Saved cluster_labels.csv ({len(self.cluster_labeled_df)} rows)") except OSError as e: logger.error(f"Could not save cluster_labels.csv: {e}") return self.cluster_labeled_df # ----------------------------------------------------------------------- # Phase implementations — original (3-6) # ----------------------------------------------------------------------- def _phase3_export_topic_table(self) -> None: if self.topic_df is None or self.topic_df.empty: raise ValueError("Phase 3: No topic_df available. Run Phase 2 first.") out_path = Path(self.config.output_dir) / 'topic_review_table.csv' out_path.parent.mkdir(parents=True, exist_ok=True) cols = ['topic_id', 'label', 'top_words', 'coherence', 'doc_count'] available_cols = [c for c in cols if c in self.topic_df.columns] export_df = self.topic_df[available_cols].sort_values('doc_count', ascending=False) export_df.to_csv(out_path, index=False) logger.info(f"Phase 3: Saved topic_review_table.csv ({len(export_df)} rows)") def _phase4_abstract_title_comparison(self) -> None: if self.df is None or self.df.empty: raise ValueError("Phase 4: No data loaded.") self.comparison_df = compare_abstract_vs_title_themes(self.df, n_topics_each=20) logger.info(f"Phase 4: Comparison complete. {len(self.comparison_df)} rows.") def _phase5_taxonomy_mapping(self) -> None: if self.topic_df is None or self.topic_df.empty: raise ValueError("Phase 5: No topic_df available.") self.topic_df = map_topics_to_pajais(self.topic_df, PAJAIS_THEMES) self.taxonomy_map = generate_taxonomy_map(self.topic_df) out_path = Path(self.config.output_dir) / 'taxonomy_map.json' out_path.parent.mkdir(parents=True, exist_ok=True) try: with open(out_path, 'w', encoding='utf-8') as f: json.dump(self.taxonomy_map, f, indent=2, default=str) logger.info("Phase 5: Saved taxonomy_map.json") except (OSError, TypeError) as e: logger.error(f"Phase 5: Could not save taxonomy_map.json: {e}") def _phase5_5_mapping_display(self) -> None: if self.topic_df is None or self.topic_df.empty: return display_cols = ['label', 'pajais_theme', 'status', 'match_score', 'doc_count'] available = [c for c in display_cols if c in self.topic_df.columns] display_df = self.topic_df[available].copy() if 'coherence' in self.topic_df.columns: display_df['publishable'] = ( (self.topic_df.get('status', '') == 'NOVEL') & (self.topic_df.get('doc_count', 0) > self.config.publishable_min_docs) & (self.topic_df.get('coherence', 0.0) > self.config.publishable_min_coherence) ) else: display_df['publishable'] = False out_path = Path(self.config.output_dir) / 'pajais_mapping.csv' display_df.to_csv(out_path, index=False) logger.info(f"Phase 5.5: Saved pajais_mapping.csv ({len(display_df)} rows)") def _phase6_narrative(self) -> None: taxonomy_map = self.taxonomy_map or {} comparison_df = self.comparison_df if self.comparison_df is not None else pd.DataFrame() topic_df = self.topic_df if self.topic_df is not None else pd.DataFrame() self.narrative = generate_section7_narrative( taxonomy_map=taxonomy_map, comparison_df=comparison_df, topic_df=topic_df, ) logger.info(f"Phase 6: Narrative generated ({len(self.narrative)} characters).") # ----------------------------------------------------------------------- # Phase 6.5 — Agentic Council (NEW) # ----------------------------------------------------------------------- def _phase6_5_agentic_council(self) -> None: """Phase 6.5: Multi-model council (Mistral + Gemini + Anthropic judge).""" if not _ADDITIONS_AVAILABLE: logger.warning("Phase 6.5: tools_additions not available; skipping council.") return if not self.taxonomy_map: raise ValueError("Phase 6.5: Run taxonomy mapping first (Phase 5).") logger.info("Phase 6.5: Convening Agentic Council…") self.council_result = run_agentic_council( taxonomy_map=self.taxonomy_map, topic_df=self.topic_df, mistral_api_key=self.config.mistral_api_key, gemini_api_key=self.config.gemini_api_key, ollama_url=self.config.ollama_url, ) # Persist council report out = Path(self.config.output_dir) / "council_report.json" try: with open(out, 'w', encoding='utf-8') as f: json.dump(self.council_result, f, indent=2, ensure_ascii=False) logger.info("Phase 6.5: Saved council_report.json") except OSError as e: logger.error(f"Phase 6.5: Could not save council_report.json: {e}") # ----------------------------------------------------------------------- # DBSCAN artifact export helper # ----------------------------------------------------------------------- def _export_dbscan_artifacts(self) -> None: out_dir = Path(self.config.output_dir) if self.cluster_df is not None and not self.cluster_df.empty: p = out_dir / "cluster_documents.csv" try: self.cluster_df.to_csv(p, index=False) self.artifacts["cluster_documents"] = str(p) logger.info(f"Exported cluster_documents.csv") except OSError as e: logger.error(f"Could not save cluster_documents.csv: {e}") if self.cluster_summary_df is not None and not self.cluster_summary_df.empty: p = out_dir / "cluster_summary.csv" try: self.cluster_summary_df.to_csv(p, index=False) self.artifacts["cluster_summary"] = str(p) logger.info(f"Exported cluster_summary.csv") except OSError as e: logger.error(f"Could not save cluster_summary.csv: {e}") # ----------------------------------------------------------------------- # Supplementary insights # ----------------------------------------------------------------------- def _discover_supplementary_insights(self) -> None: insights: Dict[str, Any] = {} try: if self.topic_df is not None and 'status' in self.topic_df.columns: novel_df = self.topic_df[self.topic_df['status'] == 'NOVEL'] if not novel_df.empty: top_novel = novel_df.sort_values('doc_count', ascending=False).iloc[0] insights['blind_spot_theme'] = { 'label': top_novel.get('label', ''), 'doc_count': int(top_novel.get('doc_count', 0)), 'top_words': top_novel.get('top_words', ''), } except Exception as e: logger.warning(f"blind_spot_theme computation failed: {e}") try: if self.df is not None and 'year' in self.df.columns: years = pd.to_numeric(self.df['year'], errors='coerce').dropna() if not years.empty and self.lda_result and self.lda_result.get('doc_topics'): year_list = years.tolist() doc_topics = self.lda_result['doc_topics'] n_docs = min(len(year_list), len(doc_topics)) year_entropy: Dict[int, List[float]] = {} for i in range(n_docs): yr = int(year_list[i]) probs = [p for _, p in doc_topics[i]] if probs: probs_arr = np.array(probs) probs_arr = probs_arr / probs_arr.sum() entropy = float(-np.sum(probs_arr * np.log(probs_arr + 1e-9))) if yr not in year_entropy: year_entropy[yr] = [] year_entropy[yr].append(entropy) if year_entropy: avg_entropy = {yr: np.mean(ents) for yr, ents in year_entropy.items()} golden_year = max(avg_entropy, key=lambda y: avg_entropy[y]) insights['golden_year'] = { 'year': golden_year, 'entropy': round(avg_entropy[golden_year], 4), } except Exception as e: logger.warning(f"golden_year computation failed: {e}") try: if self.comparison_df is not None and not self.comparison_df.empty: ab = self.comparison_df[self.comparison_df['source'] == 'abstract'][ ['label', 'doc_count'] ].rename(columns={'doc_count': 'ab_count'}) ti = self.comparison_df[self.comparison_df['source'] == 'title'][ ['label', 'doc_count'] ].rename(columns={'doc_count': 'ti_count'}) merged = ab.merge(ti, on='label', how='inner') if not merged.empty: merged['ratio'] = merged['ab_count'] / (merged['ti_count'] + 1) iceberg = merged[merged['ratio'] >= 3.0].sort_values('ratio', ascending=False) insights['iceberg_topics'] = iceberg.to_dict('records') except Exception as e: logger.warning(f"iceberg_topics computation failed: {e}") try: if self.taxonomy_map: publishable = self.taxonomy_map.get('publishable_novel_themes', []) if publishable: top_pub = max(publishable, key=lambda x: x.get('coherence', 0.0)) insights['top_publishable_gap'] = top_pub except Exception as e: logger.warning(f"top_publishable_gap computation failed: {e}") # NEW: DBSCAN stats try: if self.cluster_df is not None and not self.cluster_df.empty: n_clusters = len(set(self.cluster_df["cluster_final"]) - {-1}) n_noise = int(self.cluster_df["is_noise"].sum()) largest = self.cluster_df["cluster_final"].value_counts() largest = largest[largest.index != -1] insights['dbscan_stats'] = { 'n_clusters': n_clusters, 'n_noise': n_noise, 'largest_cluster_size': int(largest.iloc[0]) if not largest.empty else 0, } except Exception as e: logger.warning(f"dbscan_stats failed: {e}") self.supplementary_insights = insights logger.info(f"Supplementary insights computed: {list(insights.keys())}") # ----------------------------------------------------------------------- # Summary builder # ----------------------------------------------------------------------- def _build_summary(self) -> Dict[str, Any]: gap = {} if self.taxonomy_map: gap = self.taxonomy_map.get('gap_analysis', {}) return { 'success': len(self._errors) == 0, 'phases_completed': self._phases_completed, 'topic_count': len(self.topic_df) if self.topic_df is not None else 0, 'novel_count': gap.get('novel_count', 0), 'mapped_count': gap.get('mapped_count', 0), 'pajais_coverage_pct': gap.get('coverage_pct', 0.0), 'artifacts': self.artifacts, 'errors': self._errors, 'warnings': self._warnings, 'topic_df': self.topic_df, 'comparison_df': self.comparison_df, 'taxonomy_map': self.taxonomy_map, 'narrative': self.narrative, 'supplementary_insights': self.supplementary_insights, 'validation': self.validation, # NEW 'cluster_df': self.cluster_df, 'cluster_summary_df': self.cluster_summary_df, 'council_result': self.council_result, }