Update tools.py

tools.py

@@ -27,6 +27,25 @@ from gensim.models.phrases import Phrases, Phraser
 
 from tqdm import tqdm
 
+# Optional heavy deps — imported lazily inside functions
+try:
+    import torch
+    _TORCH_AVAILABLE = True
+except ImportError:
+    _TORCH_AVAILABLE = False
+
+try:
+    import umap as _umap_module
+    _UMAP_AVAILABLE = True
+except ImportError:
+    _UMAP_AVAILABLE = False
+
+try:
+    import hdbscan as _hdbscan_module
+    _HDBSCAN_AVAILABLE = True
+except ImportError:
+    _HDBSCAN_AVAILABLE = False
+
 pd.options.mode.chained_assignment = None
 
 # ---------------------------------------------------------------------------
@@ -45,6 +64,45 @@ _RE_WHITESPACE = re.compile(r'\s+')
 # NLTK Bootstrap
 # ---------------------------------------------------------------------------
 
+# =============================================================================
+# GROUP 0: TITLE + ABSTRACT COMBINED COLUMN
+# =============================================================================
+
+def build_title_abstract_column(df: pd.DataFrame) -> pd.DataFrame:
+    """Create a 'title_abstract' combined column for SPECTER2 embedding.
+
+    Concatenates title and abstract with '. ' separator.
+    Also adds 'doi_key' column: DOI if available, else 'doc_<index>'.
+
+    Args:
+        df: DataFrame with 'title' and/or 'abstract' columns.
+    Returns:
+        DataFrame copy with 'title_abstract' and 'doi_key' added.
+    """
+    df = df.copy()
+    title = df.get('title', pd.Series([''] * len(df), index=df.index)).fillna('')
+    abstract = df.get('abstract', pd.Series([''] * len(df), index=df.index)).fillna('')
+    df['title_abstract'] = (
+        title.astype(str).str.strip() + '. ' + abstract.astype(str).str.strip()
+    ).str.strip('. ').str.strip()
+
+    # DOI key: use existing DOI or generate synthetic id
+    if 'doi' in df.columns:
+        doi_filled = df['doi'].astype(str).replace({'nan': '', 'None': ''})
+        df['doi_key'] = [
+            doi if doi.strip() else f'doc_{i}'
+            for i, doi in enumerate(doi_filled)
+        ]
+    else:
+        df['doi_key'] = [f'doc_{i}' for i in range(len(df))]
+
+    logger.info(
+        f"build_title_abstract_column: {len(df)} rows, "
+        f"{(df['title_abstract'].str.len() > 10).sum()} with content."
+    )
+    return df
+
+
 def _bootstrap_nltk() -> None:
     """Silently download required NLTK packages if missing."""
     packages = [
@@ -1128,4 +1186,808 @@ def export_all_artifacts(
         logger.error(f"Failed to save narrative.txt: {e}")
 
     logger.info(f"Exported {len(artifacts)} artifacts to {output_dir}/")
-    return artifacts
+    return artifacts
+
+
+# =============================================================================
+# GROUP 8: SPECTER2 DOCUMENT EMBEDDINGS
+# =============================================================================
+
+_SPECTER2_MODEL_NAME = "allenai/specter2_base"
+_specter2_tokenizer = None
+_specter2_model = None
+
+
+def _load_specter2(device: str = 'cpu'):
+    """Lazy-load SPECTER2 model and tokenizer (module-level cache)."""
+    global _specter2_tokenizer, _specter2_model
+    if _specter2_model is not None:
+        return _specter2_tokenizer, _specter2_model
+    from transformers import AutoTokenizer, AutoModel
+    logger.info(f"Loading {_SPECTER2_MODEL_NAME} ...")
+    _specter2_tokenizer = AutoTokenizer.from_pretrained(_SPECTER2_MODEL_NAME)
+    _specter2_model = AutoModel.from_pretrained(_SPECTER2_MODEL_NAME)
+    _specter2_model.eval()
+    _specter2_model.to(device)
+    logger.info("SPECTER2 loaded.")
+    return _specter2_tokenizer, _specter2_model
+
+
+def _texts_hash(texts: List[str]) -> str:
+    import hashlib
+    blob = '||'.join(t[:300] for t in texts)
+    return hashlib.md5(blob.encode('utf-8', errors='ignore')).hexdigest()[:16]
+
+
+def _embed_batch(texts: List[str], tokenizer, model, device: str = 'cpu', max_length: int = 512) -> np.ndarray:
+    import torch
+    batch = [t if (isinstance(t, str) and t.strip()) else 'information systems' for t in texts]
+    enc = tokenizer(batch, padding=True, truncation=True, max_length=max_length, return_tensors='pt')
+    enc = {k: v.to(device) for k, v in enc.items()}
+    with torch.no_grad():
+        out = model(**enc)
+    return out.last_hidden_state[:, 0, :].cpu().numpy()
+
+
+def _tfidf_fallback_embed(texts: List[str], n_components: int = 256) -> np.ndarray:
+    """TF-IDF + TruncatedSVD fallback when SPECTER2 unavailable."""
+    from sklearn.feature_extraction.text import TfidfVectorizer
+    from sklearn.decomposition import TruncatedSVD
+    safe = [t if (isinstance(t, str) and t.strip()) else 'information' for t in texts]
+    nc = min(n_components, len(safe) - 1, 7999)
+    nc = max(nc, 2)
+    vec = TfidfVectorizer(max_features=8000, ngram_range=(1, 2), sublinear_tf=True, min_df=1)
+    mat = vec.fit_transform(safe)
+    svd = TruncatedSVD(n_components=nc, random_state=42)
+    dense = svd.fit_transform(mat).astype(np.float32)
+    norms = np.linalg.norm(dense, axis=1, keepdims=True)
+    norms[norms == 0] = 1.0
+    return dense / norms
+
+
+def embed_with_specter2(
+    texts: List[str],
+    cache_dir: str = 'outputs/specter_cache',
+    batch_size: int = 8,
+    device: str = 'cpu',
+) -> np.ndarray:
+    """Generate L2-normalised SPECTER2 (768-dim) embeddings, one per paper.
+
+    Caches results to disk keyed by MD5 of input texts to avoid re-embedding.
+    Falls back to TF-IDF + SVD (256-dim) if transformers/torch not installed.
+
+    Args:
+        texts: List of 'title. abstract' strings (from build_title_abstract_column).
+        cache_dir: Directory for .npy cache files.
+        batch_size: Texts per forward pass (lower if OOM on CPU).
+        device: 'cpu' or 'cuda'.
+    Returns:
+        np.ndarray shape (N, D), L2-normalised float32.
+    """
+    Path(cache_dir).mkdir(parents=True, exist_ok=True)
+    cache_file = Path(cache_dir) / f"{_texts_hash(texts)}.npy"
+
+    if cache_file.exists():
+        logger.info(f"embed_with_specter2: loading from cache {cache_file}")
+        return np.load(str(cache_file))
+
+    if not _TORCH_AVAILABLE:
+        logger.warning("torch not available — using TF-IDF fallback embeddings.")
+        embs = _tfidf_fallback_embed(texts)
+        np.save(str(cache_file), embs)
+        return embs
+
+    try:
+        tokenizer, model = _load_specter2(device=device)
+    except Exception as exc:
+        logger.warning(f"SPECTER2 load failed ({exc}) — using TF-IDF fallback.")
+        embs = _tfidf_fallback_embed(texts)
+        np.save(str(cache_file), embs)
+        return embs
+
+    all_embs: List[np.ndarray] = []
+    for i in tqdm(range(0, len(texts), batch_size), desc='SPECTER2'):
+        batch = texts[i: i + batch_size]
+        try:
+            all_embs.append(_embed_batch(batch, tokenizer, model, device))
+        except Exception as e:
+            logger.warning(f"Batch {i} failed ({e}); using zeros.")
+            all_embs.append(np.zeros((len(batch), 768), dtype=np.float32))
+
+    embs = np.vstack(all_embs).astype(np.float32)
+    norms = np.linalg.norm(embs, axis=1, keepdims=True)
+    norms[norms == 0] = 1.0
+    embs = embs / norms
+
+    np.save(str(cache_file), embs)
+    logger.info(f"embed_with_specter2: saved {embs.shape} embeddings → {cache_file}")
+    return embs
+
+
+# =============================================================================
+# GROUP 9: UMAP + HDBSCAN CLUSTERING  (replaces old DBSCAN pipeline)
+# =============================================================================
+
+def _run_umap(
+    embeddings: np.ndarray,
+    n_components: int = 50,
+    n_neighbors: int = 15,
+    min_dist: float = 0.0,
+    random_state: int = 42,
+) -> np.ndarray:
+    """UMAP dimensionality reduction (cosine metric). Falls back to PCA."""
+    n_comp = min(n_components, embeddings.shape[0] - 2, embeddings.shape[1])
+    n_comp = max(n_comp, 2)
+    n_neigh = min(n_neighbors, embeddings.shape[0] - 1)
+    n_neigh = max(n_neigh, 2)
+
+    if _UMAP_AVAILABLE:
+        try:
+            reducer = _umap_module.UMAP(
+                n_components=n_comp,
+                n_neighbors=n_neigh,
+                min_dist=min_dist,
+                metric='cosine',
+                random_state=random_state,
+                low_memory=True,
+            )
+            return reducer.fit_transform(embeddings)
+        except Exception as e:
+            logger.warning(f"UMAP failed ({e}); falling back to PCA.")
+
+    from sklearn.decomposition import PCA
+    pca = PCA(n_components=n_comp, random_state=random_state)
+    return pca.fit_transform(embeddings)
+
+
+def _hdbscan_labels(
+    reduced: np.ndarray,
+    min_cluster_size: int,
+    min_samples: int = 3,
+) -> np.ndarray:
+    if _HDBSCAN_AVAILABLE:
+        try:
+            clusterer = _hdbscan_module.HDBSCAN(
+                min_cluster_size=min_cluster_size,
+                min_samples=min_samples,
+                cluster_selection_method='eom',
+                metric='euclidean',
+            )
+            return clusterer.fit_predict(reduced)
+        except Exception as e:
+            logger.warning(f"HDBSCAN failed ({e}); falling back to KMeans.")
+
+    from sklearn.cluster import KMeans
+    n_c = max(3, reduced.shape[0] // max(min_cluster_size, 1))
+    n_c = min(n_c, 30)
+    return KMeans(n_clusters=n_c, random_state=42, n_init=10).fit_predict(reduced)
+
+
+def _sweep_clusters(
+    reduced: np.ndarray,
+    target_min: int = 15,
+    target_max: int = 30,
+    min_samples: int = 3,
+) -> Tuple[np.ndarray, int]:
+    """Sweep min_cluster_size until n_clusters lands in [target_min, target_max]."""
+    best_labels: Optional[np.ndarray] = None
+    best_n = 0
+    best_dist = float('inf')
+
+    # Sweep: small mcs → many clusters; large mcs → few clusters
+    for mcs in list(range(5, 51)):
+        labels = _hdbscan_labels(reduced, min_cluster_size=mcs, min_samples=min_samples)
+        n_c = len(set(labels) - {-1})
+        if target_min <= n_c <= target_max:
+            logger.info(f"Cluster sweep: min_cluster_size={mcs} → {n_c} clusters ✓")
+            return labels, mcs
+        dist = min(abs(n_c - target_min), abs(n_c - target_max))
+        if dist < best_dist:
+            best_dist = dist
+            best_labels = labels
+            best_n = n_c
+
+    logger.warning(f"Cluster sweep: could not hit [{target_min},{target_max}]; got {best_n} clusters.")
+    return best_labels, -1
+
+
+def _split_large_cluster(
+    indices: List[int],
+    embeddings: np.ndarray,
+    max_size: int,
+    next_id: int,
+) -> Dict[int, int]:
+    """Split oversized cluster using KMeans. Returns {old_idx: new_cluster_id}."""
+    from sklearn.cluster import KMeans
+    sub_embs = embeddings[indices]
+    k = max(2, len(indices) // max_size + 1)
+    k = min(k, len(indices))
+    labels = KMeans(n_clusters=k, random_state=42, n_init=5).fit_predict(sub_embs)
+    mapping: Dict[int, int] = {}
+    for local_i, orig_i in enumerate(indices):
+        mapping[orig_i] = next_id + int(labels[local_i])
+    return mapping
+
+
+def _cosine_sim_filter(
+    labels: np.ndarray,
+    embeddings: np.ndarray,
+    sim_low: float = 0.50,
+    sim_high: float = 0.60,
+) -> Tuple[np.ndarray, Dict[int, float]]:
+    """
+    Compute mean intra-cluster cosine similarity.
+    Clusters below sim_low are dissolved to noise (-1).
+    Returns updated labels and per-cluster similarity dict.
+    """
+    unique_ids = sorted(set(labels) - {-1})
+    cluster_sims: Dict[int, float] = {}
+    new_labels = labels.copy()
+
+    for cid in unique_ids:
+        mask = labels == cid
+        sub = embeddings[mask]
+        if len(sub) < 2:
+            cluster_sims[cid] = 1.0
+            continue
+        # Mean pairwise cosine sim = (sum of all dot products - n) / (n*(n-1))
+        # Since embeddings are L2-normalised, dot product = cosine similarity
+        dot = sub @ sub.T
+        n = len(sub)
+        mean_sim = float((dot.sum() - n) / max(n * (n - 1), 1))
+        cluster_sims[cid] = round(mean_sim, 4)
+
+        if mean_sim < sim_low:
+            new_labels[mask] = -1  # dissolve too-diffuse cluster
+            logger.debug(f"Cluster {cid}: sim={mean_sim:.3f} < {sim_low} → dissolved.")
+
+    return new_labels, cluster_sims
+
+
+def _renumber_labels(labels: np.ndarray) -> np.ndarray:
+    """Renumber cluster ids to 0..K-1 preserving -1 for noise."""
+    unique_ids = sorted(set(labels) - {-1})
+    remap = {old: new for new, old in enumerate(unique_ids)}
+    return np.array([remap.get(x, -1) for x in labels])
+
+
+def specter2_hdbscan_cluster_topics(
+    df: pd.DataFrame,
+    embeddings: np.ndarray,
+    min_cluster_size: int = 5,
+    max_cluster_size: int = 100,
+    target_min_clusters: int = 15,
+    target_max_clusters: int = 30,
+    cosine_sim_low: float = 0.50,
+    cosine_sim_high: float = 0.60,
+    umap_n_components: int = 50,
+    umap_n_neighbors: int = 15,
+    random_state: int = 42,
+) -> pd.DataFrame:
+    """
+    Full SPECTER2 → UMAP → HDBSCAN clustering pipeline.
+
+    Parameters
+    ----------
+    df          : DataFrame with 'title', 'abstract', 'doi_key', 'title_abstract'.
+    embeddings  : L2-normalised SPECTER2 vectors, shape (N, D).
+    min_cluster_size   : Minimum papers per cluster (default 5).
+    max_cluster_size   : Maximum papers per cluster (default 100).
+    target_min_clusters: Target minimum cluster count (default 15).
+    target_max_clusters: Target maximum cluster count (default 30).
+    cosine_sim_low / high: Acceptable intra-cluster cosine similarity range.
+    umap_n_components  : UMAP output dimensions (default 50).
+    umap_n_neighbors   : UMAP neighbourhood size (default 15).
+    random_state       : Reproducibility seed.
+
+    Returns
+    -------
+    DataFrame with columns:
+        doc_id, doi_key, title_snippet, cluster_final, is_noise,
+        intra_cluster_sim, top_kws
+    """
+    if df is None or df.empty:
+        return pd.DataFrame()
+    n = len(df)
+    assert embeddings.shape[0] == n, "embeddings row count must match df row count."
+
+    logger.info(f"UMAP: reducing {n} docs from dim {embeddings.shape[1]} → {umap_n_components}...")
+    reduced = _run_umap(embeddings, n_components=umap_n_components,
+                        n_neighbors=umap_n_neighbors, min_dist=0.0,
+                        random_state=random_state)
+
+    logger.info("HDBSCAN: sweeping for 15-30 clusters...")
+    labels, best_mcs = _sweep_clusters(
+        reduced,
+        target_min=target_min_clusters,
+        target_max=target_max_clusters,
+        min_samples=3,
+    )
+
+    # ── Enforce max cluster size (split oversized) ──
+    next_id = int(labels.max()) + 1
+    changed = True
+    while changed:
+        changed = False
+        unique_ids = sorted(set(labels) - {-1})
+        for cid in unique_ids:
+            mask_idx = [i for i, l in enumerate(labels) if l == cid]
+            if len(mask_idx) > max_cluster_size:
+                mapping = _split_large_cluster(mask_idx, embeddings, max_cluster_size, next_id)
+                for orig_i, new_cid in mapping.items():
+                    labels[orig_i] = new_cid
+                next_id += max(mapping.values()) - cid + 1
+                changed = True
+
+    # ── Enforce min cluster size (absorb undersized) ──
+    unique_ids = sorted(set(labels) - {-1})
+    sizes = {cid: int((labels == cid).sum()) for cid in unique_ids}
+    large_ids = [cid for cid, s in sizes.items() if s >= min_cluster_size]
+    for cid, s in sizes.items():
+        if s < min_cluster_size:
+            if not large_ids:
+                labels[labels == cid] = -1
+                continue
+            sub_idx = [i for i, l in enumerate(labels) if l == cid]
+            sub_embs = embeddings[sub_idx]
+            # Find nearest large cluster centroid
+            best_cid, best_sim = -1, -2.0
+            for lcid in large_ids:
+                l_mask = labels == lcid
+                centroid = embeddings[l_mask].mean(axis=0)
+                centroid /= max(np.linalg.norm(centroid), 1e-9)
+                sim = float((sub_embs @ centroid).mean())
+                if sim > best_sim:
+                    best_sim, best_cid = sim, lcid
+            for idx in sub_idx:
+                labels[idx] = best_cid
+
+    # ── Cosine similarity quality filter ──
+    labels, cluster_sims = _cosine_sim_filter(labels, embeddings, cosine_sim_low, cosine_sim_high)
+
+    # ── Final renumber ──
+    labels = _renumber_labels(labels)
+    n_final = len(set(labels) - {-1})
+    n_noise = int((labels == -1).sum())
+    logger.info(f"Clustering complete: {n_final} clusters, {n_noise} noise docs.")
+
+    # ── Build result DataFrame ──
+    titles = df.get('title', pd.Series([''] * n)).fillna('').tolist()
+    doi_keys = df.get('doi_key', pd.Series([f'doc_{i}' for i in range(n)])).tolist()
+
+    # TF-IDF keywords per doc for display
+    ta_texts = df.get('title_abstract', pd.Series([''] * n)).fillna('').tolist()
+    try:
+        from sklearn.feature_extraction.text import TfidfVectorizer as _TV
+        _v = _TV(max_features=3000, ngram_range=(1, 2), min_df=1)
+        _m = _v.fit_transform([t or ' ' for t in ta_texts])
+        _feat = _v.get_feature_names_out()
+        top_kws = []
+        for i in range(_m.shape[0]):
+            row = _m[i].toarray().flatten()
+            idx = row.argsort()[::-1][:6]
+            top_kws.append(', '.join(_feat[j] for j in idx if row[j] > 0))
+    except Exception:
+        top_kws = [''] * n
+
+    intra_sims = [cluster_sims.get(int(l), 1.0) if l != -1 else 0.0 for l in labels]
+
+    result = pd.DataFrame({
+        'doc_id': range(n),
+        'doi_key': doi_keys,
+        'title_snippet': [t[:90] + '…' if len(t) > 90 else t for t in titles],
+        'cluster_final': labels,
+        'is_noise': labels == -1,
+        'intra_cluster_sim': intra_sims,
+        'top_kws': top_kws,
+    })
+    return result
+
+
+def get_cluster_summary(cluster_df: pd.DataFrame) -> pd.DataFrame:
+    """Aggregate cluster_df into a per-cluster summary DataFrame.
+
+    Returns DataFrame with columns:
+        cluster_id, size, is_noise_cluster, top_kws, avg_sim, label
+    """
+    if cluster_df is None or cluster_df.empty:
+        return pd.DataFrame()
+    rows = []
+    for cid, grp in cluster_df.groupby('cluster_final'):
+        rows.append({
+            'cluster_id': int(cid),
+            'size': len(grp),
+            'is_noise_cluster': cid == -1,
+            'top_kws': _most_common_kw(grp['top_kws'].tolist()),
+            'avg_sim': round(float(grp['intra_cluster_sim'].mean()), 4),
+            'label': 'Unlabeled',
+        })
+    return (
+        pd.DataFrame(rows)
+        .sort_values('size', ascending=False)
+        .reset_index(drop=True)
+    )
+
+
+def _most_common_kw(kw_lists: List[str], top_n: int = 6) -> str:
+    from collections import Counter
+    counter: Counter = Counter()
+    for s in kw_lists:
+        for kw in str(s).split(','):
+            kw = kw.strip()
+            if kw:
+                counter[kw] += 1
+    return ', '.join(kw for kw, _ in counter.most_common(top_n))
+
+
+# =============================================================================
+# GROUP 10: 3-LLM CLUSTER LABELING  (Mistral + Gemini + HF Inference — all free)
+# =============================================================================
+
+import time as _time
+import httpx as _httpx
+
+
+def _get_rep_titles(
+    cluster_df: pd.DataFrame,
+    cluster_id: int,
+    embeddings: np.ndarray,
+    top_n: int = 3,
+) -> List[str]:
+    """Select top_n papers closest to cluster centroid (highest cosine sim)."""
+    mask = cluster_df['cluster_final'] == cluster_id
+    indices = cluster_df.index[mask].tolist()
+    if not indices:
+        return []
+    sub_embs = embeddings[indices]
+    centroid = sub_embs.mean(axis=0)
+    norm = np.linalg.norm(centroid)
+    if norm > 0:
+        centroid /= norm
+    sims = sub_embs @ centroid
+    top_local = np.argsort(sims)[::-1][:top_n]
+    top_global = [indices[i] for i in top_local]
+    snippets = cluster_df.loc[top_global, 'title_snippet'].tolist()
+    return [s for s in snippets if s and s.strip()]
+
+
+def _majority_label(labels: List[str]) -> str:
+    """Pick the label with most agreement among non-error strings."""
+    from collections import Counter
+    valid = [l.strip().strip('"\'') for l in labels
+             if l and not l.startswith('[') and len(l.strip()) > 2]
+    if not valid:
+        return labels[0] if labels else 'Research Cluster'
+    c = Counter(valid)
+    top_count = max(c.values())
+    winners = [l for l, cnt in c.items() if cnt == top_count]
+    return max(winners, key=len)  # prefer longer label if tied
+
+
+def _call_mistral_label(prompt: str, api_key: str) -> str:
+    try:
+        r = _httpx.post(
+            'https://api.mistral.ai/v1/chat/completions',
+            headers={'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'},
+            json={'model': 'mistral-small-latest', 'messages': [{'role': 'user', 'content': prompt}],
+                  'max_tokens': 20, 'temperature': 0.2},
+            timeout=20.0,
+        )
+        r.raise_for_status()
+        return r.json()['choices'][0]['message']['content'].strip()
+    except Exception as e:
+        return f'[Mistral: {e}]'
+
+
+def _call_gemini_label(prompt: str, api_key: str) -> str:
+    try:
+        url = (f'https://generativelanguage.googleapis.com/v1beta/models/'
+               f'gemini-2.0-flash:generateContent?key={api_key}')
+        r = _httpx.post(
+            url,
+            headers={'Content-Type': 'application/json'},
+            json={'contents': [{'parts': [{'text': prompt}]}],
+                  'generationConfig': {'maxOutputTokens': 20, 'temperature': 0.2}},
+            timeout=20.0,
+        )
+        r.raise_for_status()
+        cands = r.json().get('candidates', [])
+        return cands[0]['content']['parts'][0]['text'].strip() if cands else '[Gemini: empty]'
+    except Exception as e:
+        return f'[Gemini: {e}]'
+
+
+def _call_deepseek_label(prompt: str, api_key: str,
+                         model: str = 'deepseek-chat') -> str:
+    """DeepSeek API (OpenAI-compatible) — free $5 credit on signup."""
+    try:
+        r = _httpx.post(
+            'https://api.deepseek.com/v1/chat/completions',
+            headers={'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'},
+            json={'model': model, 'messages': [{'role': 'user', 'content': prompt}],
+                  'max_tokens': 20, 'temperature': 0.2},
+            timeout=25.0,
+        )
+        r.raise_for_status()
+        return r.json()['choices'][0]['message']['content'].strip()
+    except Exception as e:
+        return f'[DeepSeek: {e}]'
+
+
+def _label_prompt(titles: List[str]) -> str:
+    titles_block = '\n'.join(f'- {t}' for t in titles[:3])
+    return (
+        'You are an expert Information Systems researcher. '
+        'Given these 3 representative paper titles from one research cluster, '
+        'provide a concise 3-5 word thematic label. '
+        'Return ONLY the label, nothing else.\n\n'
+        f'Titles:\n{titles_block}'
+    )
+
+
+def label_clusters_3llm(
+    cluster_df: pd.DataFrame,
+    cluster_summary_df: pd.DataFrame,
+    embeddings: np.ndarray,
+    mistral_api_key: str = '',
+    gemini_api_key: str = '',
+    deepseek_api_key: str = '',
+    max_clusters: int = 30,
+) -> pd.DataFrame:
+    """Label each cluster using Mistral + Gemini + DeepSeek (all free/affordable APIs).
+
+    For each cluster:
+    1. Selects 3 most representative paper titles (closest to centroid).
+    2. Sends the same prompt to all 3 LLMs independently.
+    3. Majority vote picks the final label; all 3 candidates stored.
+
+    Falls back to keyword-based labeling if no API keys provided.
+
+    Args:
+        cluster_df        : Document-level cluster DataFrame.
+        cluster_summary_df: Cluster-level summary from get_cluster_summary().
+        embeddings        : SPECTER2 embeddings aligned with cluster_df rows.
+        mistral_api_key   : Mistral API key (free tier).
+        gemini_api_key    : Google AI Studio API key (free tier).
+        deepseek_api_key  : DeepSeek API key (free $5 credit — deepseek.com).
+        max_clusters      : Cap API calls to this many clusters.
+
+    Returns:
+        cluster_summary_df copy with 'label', 'label_mistral',
+        'label_gemini', 'label_hf' columns populated.
+    """
+    summary = cluster_summary_df.copy()
+    if 'label_mistral' not in summary.columns:
+        summary['label_mistral'] = ''
+    if 'label_gemini' not in summary.columns:
+        summary['label_gemini'] = ''
+    if 'label_hf' not in summary.columns:
+        summary['label_hf'] = ''
+
+    has_any_key = any([mistral_api_key.strip(), gemini_api_key.strip(), deepseek_api_key.strip()])
+
+    for idx, row in summary.iterrows():
+        if idx >= max_clusters:
+            break
+        cid = row['cluster_id']
+        if cid == -1:
+            summary.at[idx, 'label'] = 'Noise / Outliers'
+            continue
+
+        titles = _get_rep_titles(cluster_df, cid, embeddings, top_n=3)
+        if not titles:
+            summary.at[idx, 'label'] = row.get('top_kws', 'Unknown')[:40]
+            continue
+
+        if not has_any_key:
+            # Fallback: keyword-based label from top_kws
+            kws = str(row.get('top_kws', '')).split(',')
+            label = ' '.join(w.strip().title() for w in kws[:3] if w.strip())
+            summary.at[idx, 'label'] = label or 'Research Cluster'
+            continue
+
+        prompt = _label_prompt(titles)
+        candidates: List[str] = []
+
+        if mistral_api_key.strip():
+            ml = _call_mistral_label(prompt, mistral_api_key)
+            summary.at[idx, 'label_mistral'] = ml
+            candidates.append(ml)
+            _time.sleep(0.2)
+
+        if gemini_api_key.strip():
+            gl = _call_gemini_label(prompt, gemini_api_key)
+            summary.at[idx, 'label_gemini'] = gl
+            candidates.append(gl)
+            _time.sleep(0.2)
+
+        if deepseek_api_key.strip():
+            dl = _call_deepseek_label(prompt, deepseek_api_key)
+            summary.at[idx, 'label_deepseek'] = dl
+            candidates.append(dl)
+            _time.sleep(0.2)
+
+        summary.at[idx, 'label'] = _majority_label(candidates) if candidates else 'Research Cluster'
+        logger.info(f"Cluster {cid} labeled: {summary.at[idx, 'label']!r} "
+                    f"(candidates: {candidates})")
+
+    return summary
+
+
+# =============================================================================
+# GROUP 11: AGENTIC COUNCIL  (Mistral + Gemini + DeepSeek — DeepSeek as synthesis judge)
+# =============================================================================
+
+COUNCIL_PROMPT_TEMPLATE = """You are a senior Information Systems research analyst.
+You have been given a PAJAIS research gap analysis report with the following findings:
+{findings}
+Based on this analysis, provide your expert assessment covering:
+1. The 3 most strategically important research gaps for the field
+2. Which novel topics have the highest publication impact potential
+3. Recommended methodologies for addressing the top gap
+4. Any risks or caveats in the analysis
+Be specific, cite topic names from the report, and limit your response to 300 words."""
+
+SYNTHESIS_PROMPT_TEMPLATE = """You are the Chief Research Officer synthesizing advice from three expert panels.
+Panel A (Mistral) said:
+{mistral_response}
+Panel B (Gemini) said:
+{gemini_response}
+Panel C (DeepSeek) said:
+{deepseek_response}
+Your task:
+1. Identify the 2-3 points ALL panels AGREE on (consensus insights)
+2. Identify where they DIVERGE and explain which view is most defensible
+3. Produce a final 200-word synthesis recommendation
+Structure your response as:
+### Consensus
+<points>
+### Divergence
+<analysis>
+### Final Recommendation
+<synthesis>"""
+
+
+def _call_mistral(prompt: str, api_key: str, model: str = 'mistral-large-latest') -> str:
+    try:
+        r = _httpx.post(
+            'https://api.mistral.ai/v1/chat/completions',
+            headers={'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'},
+            json={'model': model, 'messages': [{'role': 'user', 'content': prompt}],
+                  'max_tokens': 500, 'temperature': 0.4},
+            timeout=30.0,
+        )
+        r.raise_for_status()
+        return r.json()['choices'][0]['message']['content'].strip()
+    except Exception as e:
+        logger.error(f'Mistral call failed: {e}')
+        return f'[Mistral unavailable: {e}]'
+
+
+def _call_gemini(prompt: str, api_key: str, model: str = 'gemini-2.0-flash') -> str:
+    try:
+        url = (f'https://generativelanguage.googleapis.com/v1beta/models/'
+               f'{model}:generateContent?key={api_key}')
+        r = _httpx.post(
+            url,
+            headers={'Content-Type': 'application/json'},
+            json={'contents': [{'parts': [{'text': prompt}]}],
+                  'generationConfig': {'maxOutputTokens': 500, 'temperature': 0.4}},
+            timeout=30.0,
+        )
+        r.raise_for_status()
+        cands = r.json().get('candidates', [])
+        return cands[0]['content']['parts'][0]['text'].strip() if cands else '[Gemini: empty]'
+    except Exception as e:
+        logger.error(f'Gemini call failed: {e}')
+        return f'[Gemini unavailable: {e}]'
+
+
+def _call_deepseek(
+    prompt: str,
+    api_key: str,
+    model: str = 'deepseek-chat',
+) -> str:
+    """DeepSeek API (OpenAI-compatible endpoint). Uses deepseek-chat (DeepSeek-V3)."""
+    try:
+        r = _httpx.post(
+            'https://api.deepseek.com/v1/chat/completions',
+            headers={'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'},
+            json={'model': model, 'messages': [{'role': 'user', 'content': prompt}],
+                  'max_tokens': 500, 'temperature': 0.4},
+            timeout=30.0,
+        )
+        r.raise_for_status()
+        return r.json()['choices'][0]['message']['content'].strip()
+    except Exception as e:
+        logger.error(f'DeepSeek call failed: {e}')
+        return f'[DeepSeek unavailable: {e}]'
+
+
+def run_agentic_council(
+    taxonomy_map: Dict[str, Any],
+    topic_df: Optional[pd.DataFrame],
+    mistral_api_key: str = '',
+    gemini_api_key: str = '',
+    deepseek_api_key: str = '',
+    anthropic_api_key: str = '',   # kept for backward compat — not used
+) -> Dict[str, str]:
+    """Run 4-stage agentic council: Mistral → Gemini → DeepSeek panels + DeepSeek synthesis.
+
+    Returns dict with keys:
+        'findings_summary', 'mistral', 'gemini', 'deepseek', 'synthesis'
+    """
+    gap = taxonomy_map.get('gap_analysis', {})
+    novel_themes = taxonomy_map.get('novel_themes', [])[:5]
+    pub_themes = taxonomy_map.get('publishable_novel_themes', [])[:3]
+
+    novel_str = '\n'.join(
+        f"  - {t['label']} (n={t['doc_count']}, coherence={t['coherence']:.2f})"
+        for t in novel_themes
+    )
+    pub_str = '\n'.join(
+        f"  - {t['label']} (n={t['doc_count']}, coherence={t['coherence']:.2f})"
+        for t in pub_themes
+    )
+    covered_str = ', '.join(gap.get('covered_themes', [])[:5])
+    uncovered_str = ', '.join(gap.get('uncovered_themes', [])[:5])
+    top_topics_str = ''
+    if topic_df is not None and not topic_df.empty and 'label' in topic_df.columns:
+        top_topics_str = ', '.join(topic_df.head(5)['label'].tolist())
+
+    findings = (
+        f"PAJAIS Coverage: {gap.get('coverage_pct', 0):.1f}% "
+        f"({gap.get('mapped_count', 0)} mapped, {gap.get('novel_count', 0)} novel)\n"
+        f"Covered themes (sample): {covered_str}\n"
+        f"Uncovered themes (sample): {uncovered_str}\n"
+        f"Top discovered topics: {top_topics_str}\n"
+        f"Novel research themes (top 5):\n{novel_str}\n"
+        f"Publishable gap candidates:\n{pub_str}"
+    ).strip()
+
+    council_prompt = COUNCIL_PROMPT_TEMPLATE.format(findings=findings)
+
+    # ── Stage 1: Mistral panel ──
+    logger.info('Council: calling Mistral (Panel A)…')
+    mistral_resp = (
+        _call_mistral(council_prompt, mistral_api_key)
+        if mistral_api_key.strip()
+        else '[Mistral API key not provided]'
+    )
+
+    # ── Stage 2: Gemini panel ──
+    logger.info('Council: calling Gemini (Panel B)…')
+    gemini_resp = (
+        _call_gemini(council_prompt, gemini_api_key)
+        if gemini_api_key.strip()
+        else '[Gemini API key not provided]'
+    )
+
+    # ── Stage 3: DeepSeek panel ──
+    logger.info('Council: calling DeepSeek (Panel C)…')
+    deepseek_resp = (
+        _call_deepseek(council_prompt, deepseek_api_key)
+        if deepseek_api_key.strip()
+        else '[DeepSeek API key not provided]'
+    )
+
+    # ── Stage 4: DeepSeek synthesis judge ──
+    synthesis_prompt = SYNTHESIS_PROMPT_TEMPLATE.format(
+        mistral_response=mistral_resp,
+        gemini_response=gemini_resp,
+        deepseek_response=deepseek_resp,
+    )
+    logger.info('Council: calling DeepSeek (synthesis judge)…')
+    synthesis_resp = (
+        _call_deepseek(synthesis_prompt, deepseek_api_key, model='deepseek-reasoner')
+        if deepseek_api_key.strip()
+        else '[DeepSeek API key not provided — synthesis skipped]'
+    )
+
+    return {
+        'findings_summary': findings,
+        'mistral': mistral_resp,
+        'gemini': gemini_resp,
+        'deepseek': deepseek_resp,
+        'synthesis': synthesis_resp,
+    }