|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| import re
|
| import json
|
| import os
|
| import itertools
|
| import numpy as np
|
| import pandas as pd
|
| import plotly.express as px
|
| import plotly.graph_objects as go
|
| from langchain_core.tools import tool
|
| from langchain_core.prompts import PromptTemplate
|
| from langchain_core.output_parsers import JsonOutputParser
|
| from langchain_mistralai import ChatMistralAI
|
| from langchain_groq import ChatGroq
|
| from langchain_google_genai import ChatGoogleGenerativeAI
|
| from sentence_transformers import SentenceTransformer
|
|
|
|
|
| _label_sim_model = None
|
| from sklearn.cluster import AgglomerativeClustering, DBSCAN
|
| from sklearn.metrics.pairwise import cosine_similarity
|
| from sklearn.decomposition import PCA
|
| import nltk
|
| import torch
|
| from transformers import AutoTokenizer
|
| from adapters import AutoAdapterModel
|
| import umap
|
| import hdbscan
|
|
|
| nltk.download("punkt", quiet=True)
|
| nltk.download("punkt_tab", quiet=True)
|
| from nltk.tokenize import sent_tokenize
|
|
|
|
|
|
|
|
|
| RUN_CONFIGS = {
|
| "abstract": ["Abstract"],
|
| "title": ["Title"],
|
| "combined": ["Combined"],
|
| }
|
|
|
| MODEL_NAME = "all-MiniLM-L6-v2"
|
| NEAREST_K = 5
|
| MAX_LABEL_TOPICS = 60
|
| MAX_SENTENCES = 3000
|
| DEFAULT_THRESHOLD = 0.7
|
| MISTRAL_TIMEOUT = 120
|
|
|
| BOILERPLATE_PATTERNS = [
|
| r"©\s*\d{4}",
|
| r"elsevier\s*(b\.v\.)?",
|
| r"springer\s*(nature)?",
|
| r"wiley\s*(online\s*library)?",
|
| r"all\s+rights\s+reserved",
|
| r"published\s+by\s+[a-z\s]+",
|
| r"doi:\s*10\.",
|
| r"www\.[a-z]+\.[a-z]+",
|
| r"https?://",
|
| r"copyright\s*\d{4}",
|
| r"taylor\s*&\s*francis",
|
| r"sage\s+publications",
|
| r"emerald\s+publishing",
|
| r"journal\s+of\s+[a-z\s]+issn",
|
| r"volume\s+\d+,?\s+issue\s+\d+",
|
| r"pp\.\s*\d+[-–]\d+",
|
| r"received\s+\d+\s+\w+\s+\d{4}",
|
| r"accepted\s+\d+\s+\w+\s+\d{4}",
|
| r"available\s+online",
|
| r"this\s+is\s+an\s+open\s+access",
|
| r"creative\s+commons",
|
| r"please\s+cite\s+this\s+article",
|
| ]
|
|
|
| PAJAIS_TAXONOMY = [
|
| "Artificial Intelligence Methods",
|
| "Natural Language Processing",
|
| "Machine Learning",
|
| "Deep Learning",
|
| "Knowledge Representation",
|
| "Ontologies & Semantic Web",
|
| "Information Retrieval",
|
| "Recommender Systems",
|
| "Decision Support Systems",
|
| "Human-Computer Interaction",
|
| "Explainability & Transparency",
|
| "Fairness, Accountability & Ethics",
|
| "Data Management & Integration",
|
| "Text Mining & Analytics",
|
| "Sentiment Analysis",
|
| "Social Media Analysis",
|
| "Business Intelligence",
|
| "Process Automation & RPA",
|
| "Computer Vision",
|
| "Speech & Audio Processing",
|
| "Multi-Agent Systems",
|
| "Robotics & Autonomous Systems",
|
| "Healthcare & Biomedical AI",
|
| "Finance & Risk Analytics",
|
| "Education & E-Learning",
|
| ]
|
|
|
|
|
|
|
|
|
|
|
| def _is_boilerplate(s: str) -> bool:
|
| return any(map(lambda p: bool(re.search(p, s, re.IGNORECASE)), BOILERPLATE_PATTERNS))
|
|
|
|
|
| def _clean_sentences(raw: list) -> list:
|
| long_enuf = list(filter(lambda s: len(s.split()) >= 6, raw))
|
| return long_enuf
|
|
|
|
|
| def _texts_to_sentences(texts: list) -> list:
|
| return _clean_sentences(texts)
|
|
|
|
|
| def _embed(sentences: list) -> np.ndarray:
|
| print(f"Loading SPECTER2 for {len(sentences)} items...")
|
| tokenizer = AutoTokenizer.from_pretrained('allenai/specter2_base')
|
| model = AutoAdapterModel.from_pretrained('allenai/specter2_base')
|
| model.load_adapter("allenai/specter2", source="hf", load_as="proximity", set_active=True)
|
|
|
| device = "cuda" if torch.cuda.is_available() else "cpu"
|
| model.to(device)
|
| model.eval()
|
|
|
| batch_size = 8
|
|
|
| def _process_batch(i):
|
| batch = sentences[i:i+batch_size]
|
| inputs = tokenizer(batch, padding=True, truncation=True, return_tensors="pt", return_token_type_ids=False, max_length=512)
|
| inputs = {k: v.to(device) for k, v in inputs.items()}
|
| return model(**inputs).last_hidden_state[:, 0, :].cpu().numpy()
|
|
|
| with torch.no_grad():
|
| all_embeddings = list(map(_process_batch, range(0, len(sentences), batch_size)))
|
|
|
| return np.vstack(all_embeddings) if all_embeddings else np.array([])
|
|
|
|
|
| def _cluster(embeddings: np.ndarray, threshold: float) -> np.ndarray:
|
| return AgglomerativeClustering(
|
| metric="cosine", linkage="average",
|
| distance_threshold=threshold, n_clusters=None,
|
| ).fit_predict(embeddings)
|
|
|
|
|
| def _compute_centroids(embeddings: np.ndarray, labels: np.ndarray) -> dict:
|
| valid = sorted(set(labels.tolist()) - {-1})
|
| return dict(map(lambda l: (l, embeddings[labels == l].mean(axis=0)), valid))
|
|
|
|
|
| def _nearest_sents(centroid: np.ndarray, sentences: list,
|
| embeddings: np.ndarray, k: int) -> list:
|
| sims = cosine_similarity([centroid], embeddings)[0]
|
| idxs = np.argsort(sims)[::-1][:k].tolist()
|
| return list(map(lambda i: sentences[i], idxs))
|
|
|
|
|
| def _build_summaries(labels: np.ndarray, sentences: list,
|
| embeddings: np.ndarray) -> list:
|
| centroids = _compute_centroids(embeddings, labels)
|
|
|
| def _one(tid):
|
| mask = labels == tid
|
| return {
|
| "topic_id": tid,
|
| "count": int(mask.sum()),
|
| "centroid": centroids[tid].tolist(),
|
| "nearest_sentences": _nearest_sents(
|
| centroids[tid], sentences, embeddings, NEAREST_K),
|
| }
|
| return list(map(_one, sorted(centroids.keys())))
|
|
|
|
|
| def _get_llm() -> ChatMistralAI:
|
| return ChatMistralAI(
|
| model="mistral-large-latest",
|
| temperature=0.2,
|
| timeout=MISTRAL_TIMEOUT,
|
| max_retries=0,
|
| )
|
|
|
|
|
|
|
|
|
|
|
| @tool
|
| def load_scopus_csv(file_path: str) -> str:
|
| """
|
| Load a Scopus CSV file correctly.
|
| Uses utf-8-sig (handles BOM) + quoting=0 (respects quoted multi-line cells).
|
| """
|
| df = pd.read_csv(
|
| file_path,
|
| encoding="utf-8-sig",
|
| quoting=0,
|
| engine="python",
|
| on_bad_lines="skip",
|
| )
|
| df.to_csv("loaded_data.csv", index=False, encoding="utf-8")
|
|
|
| n = len(df)
|
| cols = list(df.columns)
|
|
|
| abs_texts = list(df["Abstract"].dropna().astype(str)) if "Abstract" in cols else []
|
| ttl_texts = list(df["Title"].dropna().astype(str)) if "Title" in cols else []
|
|
|
| df["Combined"] = df["Title"].fillna("") + " " + df["Abstract"].fillna("")
|
| df.to_csv("loaded_data.csv", index=False, encoding="utf-8")
|
| combined_texts = list(df["Combined"].astype(str))
|
|
|
| abs_sents = combined_texts
|
| ttl_sents = ttl_texts
|
|
|
| years = pd.to_numeric(df["Year"], errors="coerce").dropna() if "Year" in cols else pd.Series([], dtype=float)
|
| year_range = f"{int(years.min())} – {int(years.max())}" if len(years) else "N/A"
|
|
|
| return json.dumps({
|
| "papers": n,
|
| "abstract_sentences": len(abs_sents),
|
| "title_sentences": len(ttl_sents),
|
| "year_range": year_range,
|
| "columns": cols,
|
| "abstract_coverage_pct": round(len(abs_texts) / n * 100, 1) if n else 0,
|
| "title_coverage_pct": round(len(ttl_texts) / n * 100, 1) if n else 0,
|
| "sample_titles": list(df["Title"].dropna().head(5)) if "Title" in cols else [],
|
| "file_saved": "loaded_data.csv",
|
| "note": f"Sentence cap for clustering is {MAX_SENTENCES} (to optimize math processing).",
|
| }, indent=2)
|
|
|
|
|
|
|
|
|
|
|
| @tool
|
| def run_dbscan_discovery(run_key: str = "abstract") -> str:
|
| """
|
| Core clustering tool using HDBSCAN + UMAP.
|
| Embeds with Specter2, clusters using density, and saves to summaries_{run_key}.json.
|
| """
|
| df = pd.read_csv("loaded_data.csv")
|
| col = RUN_CONFIGS[run_key][0]
|
| texts = list(df[col].dropna().astype(str))
|
|
|
| all_sentences = _texts_to_sentences(texts)
|
| sentences = all_sentences[:MAX_SENTENCES]
|
| print(f"[run_dbscan_discovery] {len(all_sentences)} items → capped to {len(sentences)}")
|
|
|
| embeddings = _embed(sentences)
|
| np.save(f"emb_{run_key}.npy", embeddings)
|
|
|
|
|
| params = list(itertools.product([5, 10, 15], [5, 10, 15], [0.4, 0.5, 0.6]))
|
|
|
| def _eval_params(p):
|
| n_n, m_c, eps = p
|
| umap_emb = umap.UMAP(n_neighbors=n_n, n_components=5, metric='cosine', random_state=42).fit_transform(embeddings)
|
| labels = hdbscan.HDBSCAN(min_cluster_size=m_c, cluster_selection_epsilon=eps, metric='euclidean').fit_predict(umap_emb)
|
| n_clusters = len(set(labels) - {-1})
|
| return (15 <= n_clusters <= 30, labels, {'n_neighbors': n_n, 'min_cluster_size': m_c, 'eps': eps})
|
|
|
| valid_configs = filter(lambda x: x[0], map(_eval_params, params))
|
|
|
| def _fallback():
|
| umap_emb = umap.UMAP(n_neighbors=10, n_components=5, metric='cosine', random_state=42).fit_transform(embeddings)
|
| labels = hdbscan.HDBSCAN(min_cluster_size=5, cluster_selection_epsilon=0.5, metric='euclidean').fit_predict(umap_emb)
|
| return (True, labels, {'fallback': True})
|
|
|
|
|
| best_config = next(valid_configs, _fallback())
|
| db_labels, best_params = best_config[1], best_config[2]
|
|
|
| valid_ids = sorted(set(db_labels.tolist()) - {-1})
|
| centroids = _compute_centroids(embeddings, db_labels)
|
|
|
| def _dbscan_summary(cid):
|
| mask = db_labels == cid
|
| return {
|
| "topic_id": int(cid),
|
| "count": int(mask.sum()),
|
| "centroid": centroids[cid].tolist(),
|
| "nearest_sentences": _nearest_sents(centroids[cid], sentences, embeddings, min(5, len(sentences))),
|
| }
|
|
|
| summaries = list(map(_dbscan_summary, valid_ids))
|
|
|
| with open(f"summaries_{run_key}.json", "w", encoding="utf-8") as f:
|
| json.dump(summaries, f, indent=2)
|
|
|
|
|
| n_comp = min(2, len(embeddings), embeddings.shape[1])
|
| pca2 = PCA(n_components=n_comp).fit_transform(embeddings)
|
| fig1 = px.scatter(
|
| x=pca2[:, 0].tolist(), y=pca2[:, 1].tolist() if n_comp > 1 else [0.0]*len(pca2),
|
| color=list(map(str, db_labels.tolist())),
|
| title=f"DBSCAN Cluster Map ({run_key})", opacity=0.7,
|
| )
|
| fig1.update_layout(template="plotly_dark")
|
| chart1 = f"chart_{run_key}_dbscan_scatter.html"
|
| fig1.write_html(chart1, include_plotlyjs="cdn")
|
|
|
|
|
| counts = list(map(lambda s: s["count"], summaries))
|
| ids = list(map(lambda s: f"C{s['topic_id']}", summaries))
|
| fig2 = px.bar(x=ids, y=counts, title=f"DBSCAN Cluster Sizes ({run_key})", color=counts, color_continuous_scale="Teal")
|
| fig2.update_layout(template="plotly_dark")
|
| chart2 = f"chart_{run_key}_dbscan_bars.html"
|
| fig2.write_html(chart2, include_plotlyjs="cdn")
|
|
|
|
|
| centroid_matrix = np.array(list(map(lambda cid: centroids[cid], valid_ids)))
|
| pca_inter = PCA(n_components=2).fit_transform(centroid_matrix)
|
| cluster_sizes = list(map(lambda cid: int((db_labels == cid).sum()), valid_ids))
|
| fig3 = go.Figure(go.Scatter(
|
| x=pca_inter[:, 0].tolist(),
|
| y=pca_inter[:, 1].tolist(),
|
| mode="markers+text",
|
| marker=dict(
|
| size=list(map(lambda s: max(20, min(80, s // 2)), cluster_sizes)),
|
| color=list(range(len(valid_ids))),
|
| colorscale="Blues",
|
| opacity=0.75,
|
| sizemode="diameter",
|
| colorbar=dict(title="No. of Docs", thickness=15, len=0.7),
|
| ),
|
| text=list(map(lambda cid: f"C{cid}", valid_ids)),
|
| textposition="middle center",
|
| textfont=dict(color="white", size=9),
|
| hovertext=list(map(lambda x: f"Cluster {x[0]}<br>Size: {x[1]} docs", zip(valid_ids, cluster_sizes))),
|
| hoverinfo="text",
|
| ))
|
| fig3.update_layout(
|
| title=f"Intertopic Distance Map ({run_key})",
|
| template="plotly_dark",
|
| xaxis=dict(visible=True, title="PC1", showgrid=True, zeroline=False),
|
| yaxis=dict(visible=True, title="PC2", showgrid=True, zeroline=False),
|
| showlegend=False,
|
| )
|
| chart3 = f"chart_{run_key}_intertopic.html"
|
| fig3.write_html(chart3, include_plotlyjs="cdn")
|
|
|
| return json.dumps({
|
| "run_key": run_key,
|
| "n_clusters": len(summaries),
|
| "noise_points": int((db_labels == -1).sum()),
|
| "charts": [chart1, chart2, chart3],
|
| }, indent=2)
|
|
|
|
|
|
|
|
|
|
|
| @tool
|
| def label_topics_with_council(run_key: str = "abstract") -> str:
|
| """
|
| Main labeling tool using the AI Council (Mistral + Groq + Gemini).
|
| Reads summaries, debates labels, and saves directly to labels_{run_key}.json.
|
| """
|
| summary_file = f"summaries_{run_key}.json"
|
| assert os.path.exists(summary_file), f"ERROR: '{summary_file}' not found. Wait for run_dbscan_discovery to finish."
|
|
|
| with open(summary_file, encoding="utf-8") as f:
|
| summaries = json.load(f)
|
|
|
| top = summaries[:MAX_LABEL_TOPICS]
|
|
|
| def _format_prompt_topic(s):
|
| sents = list(map(lambda sent: sent[:500] + "...", s.get("nearest_sentences", [])[:3]))
|
| return {"topic_id": s["topic_id"], "sentences": sents}
|
|
|
| topics_for_prompt = list(map(_format_prompt_topic, top))
|
|
|
| llm_a = _get_llm()
|
| llm_b = _get_council_llm_b()
|
| llm_c = _get_council_llm_c()
|
|
|
| tmpl = (
|
| "You are an expert thematic analyst reviewing DBSCAN clusters.\n"
|
| "Clusters:\n{topics_json}\n\n"
|
| "For EACH cluster, propose a concise label (3-6 words).\n"
|
| "Return ONLY a valid JSON array. Each element: {{\"topic_id\": int, \"label\": \"...\", \"reasoning\": \"...\"}}"
|
| )
|
| prompt = PromptTemplate(input_variables=["topics_json"], template=tmpl)
|
| parser = JsonOutputParser()
|
|
|
| from langchain_core.runnables import RunnableLambda
|
|
|
|
|
| fallback_fn = RunnableLambda(
|
| lambda _: list(map(
|
| lambda s: {"topic_id": s["topic_id"], "label": "API Error/Timeout", "reasoning": "Model failed to generate JSON."},
|
| top
|
| ))
|
| )
|
|
|
| chain_a = (prompt | llm_a | parser).with_fallbacks([fallback_fn])
|
| chain_b = (prompt | llm_b | parser).with_fallbacks([fallback_fn])
|
| chain_c = (prompt | llm_c | parser).with_fallbacks([fallback_fn])
|
|
|
| input_data = {"topics_json": json.dumps(topics_for_prompt, indent=2)}
|
|
|
| res_a = chain_a.invoke(input_data)
|
| res_b = chain_b.invoke(input_data)
|
| res_c = chain_c.invoke(input_data)
|
|
|
| idx_a = {str(r["topic_id"]): r for r in res_a}
|
| idx_b = {str(r["topic_id"]): r for r in res_b}
|
| idx_c = {str(r["topic_id"]): r for r in res_c}
|
|
|
| def _consensus(s):
|
| cid = str(s["topic_id"])
|
| ra, rb, rc = idx_a.get(cid, {}), idx_b.get(cid, {}), idx_c.get(cid, {})
|
| label_a, label_b, label_c = ra.get("label", "Unknown"), rb.get("label", "Unknown"), rc.get("label", "Unknown")
|
|
|
| s_ab, s_bc, s_ca = _council_agreement_score(label_a, label_b), _council_agreement_score(label_b, label_c), _council_agreement_score(label_c, label_a)
|
| max_score = max(s_ab, s_bc, s_ca)
|
| agreed = max_score >= 0.65
|
|
|
| avg_a, avg_b, avg_c = (s_ab + s_ca) / 2, (s_ab + s_bc) / 2, (s_bc + s_ca) / 2
|
|
|
|
|
| scores = [(avg_a, label_a), (avg_b, label_b), (avg_c, label_c)]
|
| best_label = max(scores, key=lambda x: x[0])[1]
|
|
|
| consensus = best_label if agreed else label_a
|
|
|
| ui = format_consensus_ui(label_a, label_b, label_c, agreed, max_score, ra.get("reasoning",""), rb.get("reasoning",""), rc.get("reasoning",""))
|
| return {**s, "label": consensus, "council_ui": ui, "agreement_score": max_score}
|
|
|
| council_labels = list(map(_consensus, top))
|
| out = f"labels_{run_key}.json"
|
| with open(out, "w", encoding="utf-8") as f:
|
| json.dump(council_labels, f, indent=2)
|
|
|
| return json.dumps({"run_key": run_key, "total_labelled": len(council_labels), "output_file": out}, indent=2)
|
|
|
|
|
|
|
|
|
|
|
| @tool
|
| def consolidate_into_themes(run_key: str = "abstract", theme_map: str = "") -> str:
|
| """
|
| Merge topic clusters into core themes using a dual-LLM AI Council.
|
| """
|
| with open(f"labels_{run_key}.json", encoding="utf-8") as f:
|
| labelled = json.load(f)
|
|
|
| with open(f"summaries_{run_key}.json", encoding="utf-8") as f:
|
| summaries = json.load(f)
|
|
|
| sum_dict = dict(map(lambda s: (s["topic_id"], s), summaries))
|
|
|
| llm_a = _get_llm()
|
| llm_b = _get_council_llm_b()
|
| llm_c = _get_council_llm_c()
|
| parser = JsonOutputParser()
|
|
|
| prompt = PromptTemplate(
|
| input_variables=["topics_json"],
|
| template=(
|
| "You are a thematic analyst.\n\n"
|
| "Topics: {topics_json}\n\n"
|
| "Consolidate into 4-8 themes. Return JSON array. Each element: "
|
| "{{\"theme_name\": \"...\", \"topic_ids\": [1,2,3], \"rationale\": \"...\"}}"
|
| ),
|
| )
|
|
|
| from langchain_core.runnables import RunnableLambda
|
| fallback_fn = RunnableLambda(
|
| lambda _: [{"theme_name": "API Error Theme", "topic_ids": [], "rationale": "API Timeout"}]
|
| )
|
|
|
| chain_a = (prompt | llm_a | parser).with_fallbacks([fallback_fn])
|
| chain_b = (prompt | llm_b | parser).with_fallbacks([fallback_fn])
|
| chain_c = (prompt | llm_c | parser).with_fallbacks([fallback_fn])
|
|
|
| summary_input = json.dumps(list(map(lambda t: {"id": t["topic_id"], "lbl": t["label"]}, labelled)), indent=2)
|
| raw_a = chain_a.invoke({"topics_json": summary_input}) or []
|
| raw_b = chain_b.invoke({"topics_json": summary_input}) or []
|
| raw_c = chain_c.invoke({"topics_json": summary_input}) or []
|
|
|
| l_a = ", ".join(map(lambda x: x.get("theme_name", ""), raw_a[:2]))
|
| l_b = ", ".join(map(lambda x: x.get("theme_name", ""), raw_b[:2]))
|
| l_c = ", ".join(map(lambda x: x.get("theme_name", ""), raw_c[:2]))
|
|
|
| s_ab = _council_agreement_score(l_a, l_b)
|
| s_bc = _council_agreement_score(l_b, l_c)
|
| s_ca = _council_agreement_score(l_c, l_a)
|
|
|
| score = max(s_ab, s_bc, s_ca)
|
| agreed = score >= 0.3
|
| ui = format_consensus_ui(l_a, l_b, l_c, agreed, score)
|
|
|
| def _enrich_theme(theme):
|
| t_ids = theme.get("topic_ids", [])
|
|
|
| total_sents = sum(map(lambda tid: sum_dict.get(tid, {}).get("count", 0), t_ids))
|
|
|
| sents_nested = list(map(lambda tid: sum_dict.get(tid, {}).get("nearest_sentences", []), t_ids))
|
| all_sents = list(itertools.chain.from_iterable(sents_nested))
|
|
|
| const_labels = list(map(lambda l: l["label"], filter(lambda l: l["topic_id"] in t_ids, labelled)))
|
|
|
| return {
|
| **theme,
|
| "total_sentences": total_sents,
|
| "representative_sentences": all_sents[:3] if all_sents else ["Evidence pending."],
|
| "constituent_labels": const_labels,
|
| "council_ui": ui
|
| }
|
|
|
| themes = list(map(_enrich_theme, raw_a))
|
|
|
| out = f"themes_{run_key}.json"
|
| with open(out, "w", encoding="utf-8") as f:
|
| json.dump(themes, f, indent=2)
|
| with open("themes.json", "w", encoding="utf-8") as f:
|
| json.dump(themes, f, indent=2)
|
|
|
| return json.dumps({
|
| "run_key": run_key,
|
| "total_themes": len(themes),
|
| "output_file": out,
|
| "themes_preview": themes[:3],
|
| }, indent=2)
|
|
|
|
|
|
|
|
|
|
|
| @tool
|
| def compare_with_taxonomy(run_key: str = "abstract") -> str:
|
| """
|
| Map each consolidated theme to the PAJAIS 25-category taxonomy via Mistral.
|
| Returns MAPPED vs NOVEL per theme. Saves taxonomy_map.json.
|
| """
|
| run_themes_file = f"themes_{run_key}.json"
|
| themes_file = run_themes_file if os.path.exists(run_themes_file) else "themes.json"
|
| with open(themes_file, encoding="utf-8") as f:
|
| themes = json.load(f)
|
|
|
| llm = _get_llm()
|
| parser = JsonOutputParser()
|
|
|
| prompt = PromptTemplate(
|
| input_variables=["themes_json", "taxonomy"],
|
| template=(
|
| "You are a research classification expert.\n\n"
|
| "PAJAIS Taxonomy (25 categories):\n{taxonomy}\n\n"
|
| "Themes from corpus:\n{themes_json}\n\n"
|
| "For each theme, find the best PAJAIS category match.\n"
|
| "Return ONLY a valid JSON array — no markdown. Each element:\n"
|
| " theme_name: string (match input exactly)\n"
|
| " pajais_match: best PAJAIS category, or 'NOVEL' if none fits\n"
|
| " match_confidence: float 0.0-1.0\n"
|
| " reasoning: one sentence\n"
|
| " is_novel: boolean\n"
|
| ),
|
| )
|
| chain = prompt | llm | parser
|
|
|
| theme_summaries = list(map(
|
| lambda t: {
|
| "theme_name": t["theme_name"],
|
| "total_sentences": t.get("total_sentences", 0),
|
| "constituent_labels": t.get("constituent_labels", []),
|
| "sample": (t.get("representative_sentences", [""])[0][:100]
|
| if t.get("representative_sentences") else ""),
|
| },
|
| themes,
|
| ))
|
|
|
| formatted_taxonomy = "\n".join(map(lambda x: f"{x[0]+1}. {x[1]}", enumerate(PAJAIS_TAXONOMY)))
|
|
|
| mapping = chain.invoke({
|
| "themes_json": json.dumps(theme_summaries, indent=2),
|
| "taxonomy": formatted_taxonomy,
|
| })
|
|
|
| with open("taxonomy_map.json", "w", encoding="utf-8") as f:
|
| json.dump(mapping, f, indent=2)
|
|
|
| novel_count = len(list(filter(lambda m: m.get("is_novel", False), mapping)))
|
|
|
| return json.dumps({
|
| "run_key": run_key,
|
| "total_themes_mapped": len(mapping),
|
| "novel_themes": novel_count,
|
| "mapped_themes": len(mapping) - novel_count,
|
| "output_file": "taxonomy_map.json",
|
| "mapping": mapping,
|
| }, indent=2)
|
|
|
|
|
|
|
|
|
|
|
| @tool
|
| def generate_comparison_csv() -> str:
|
| """
|
| Load themes from both abstract and title runs, create side-by-side
|
| comparison DataFrame. Saves comparison.csv.
|
| """
|
| def _load(rk):
|
| p = f"themes_{rk}.json"
|
| raw = open(p, encoding="utf-8").read() if os.path.exists(p) else "[]"
|
| return json.loads(raw)
|
|
|
| abs_themes = _load("abstract")
|
| ttl_themes = _load("title")
|
| max_rows = max(len(abs_themes), len(ttl_themes), 1)
|
|
|
| pad_abs = abs_themes + [{}] * (max_rows - len(abs_themes))
|
| pad_ttl = ttl_themes + [{}] * (max_rows - len(ttl_themes))
|
|
|
| rows = list(map(
|
| lambda pair: {
|
| "#": pair[0] + 1,
|
| "Abstract Theme": pair[1][0].get("theme_name", ""),
|
| "Abstract Sents": pair[1][0].get("total_sentences", 0),
|
| "Abstract Labels": ", ".join(pair[1][0].get("constituent_labels", [])[:3]),
|
| "Title Theme": pair[1][1].get("theme_name", ""),
|
| "Title Sents": pair[1][1].get("total_sentences", 0),
|
| "Title Labels": ", ".join(pair[1][1].get("constituent_labels", [])[:3]),
|
| "Convergence": (
|
| "✓" if pair[1][0].get("theme_name", "").lower()[:8]
|
| == pair[1][1].get("theme_name", "").lower()[:8]
|
| else ""
|
| ),
|
| },
|
| enumerate(zip(pad_abs, pad_ttl)),
|
| ))
|
|
|
| df = pd.DataFrame(rows)
|
| df.to_csv("comparison.csv", index=False)
|
|
|
| return json.dumps({
|
| "output_file": "comparison.csv",
|
| "row_count": len(df),
|
| "preview": rows[:3],
|
| }, indent=2)
|
|
|
|
|
|
|
|
|
|
|
| @tool
|
| def export_narrative(run_key: str = "abstract") -> str:
|
| """
|
| Generate a 500-word Section 7 narrative using Mistral LLM.
|
| Covers methodology, themes, PAJAIS alignment, limitations, implications.
|
| Saves narrative.txt.
|
| """
|
| with open("themes.json", encoding="utf-8") as f:
|
| themes = json.load(f)
|
|
|
| tax_raw = open("taxonomy_map.json", encoding="utf-8").read() if os.path.exists("taxonomy_map.json") else "[]"
|
| tax_data = json.loads(tax_raw)
|
|
|
| llm = _get_llm()
|
| llm.temperature = 0.4
|
| prompt = PromptTemplate(
|
| input_variables=["run_key", "themes_json", "taxonomy_json"],
|
| template=(
|
| "You are writing Section 7 of an academic literature review paper.\n\n"
|
| "Analysis column: {run_key}\n"
|
| "Themes:\n{themes_json}\n\n"
|
| "PAJAIS Mapping:\n{taxonomy_json}\n\n"
|
| "Write a 500-word Section 7 covering:\n"
|
| "1. Methodology (BERTopic + Braun & Clarke 2006 six phases)\n"
|
| "2. Key themes discovered (reference each by name)\n"
|
| "3. PAJAIS taxonomy alignment (MAPPED vs NOVEL themes)\n"
|
| "4. Limitations of this computational approach\n"
|
| "5. Implications for future research\n\n"
|
| "Academic third-person prose, full paragraphs only, minimum 500 words."
|
| ),
|
| )
|
| chain = prompt | llm
|
| response = chain.invoke({
|
| "run_key": run_key,
|
| "themes_json": json.dumps(themes, indent=2),
|
| "taxonomy_json": json.dumps(tax_data, indent=2),
|
| })
|
| text = response.content if hasattr(response, "content") else str(response)
|
|
|
| with open("narrative.txt", "w", encoding="utf-8") as f:
|
| f.write(text)
|
|
|
| return json.dumps({
|
| "output_file": "narrative.txt",
|
| "word_count": len(text.split()),
|
| "preview": text[:500],
|
| }, indent=2)
|
|
|
|
|
|
|
|
|
|
|
| def _get_council_llm_b() -> ChatGroq:
|
| """Return the Groq Llama-3 model as the second council LLM."""
|
| return ChatGroq(model="llama-3.3-70b-versatile", temperature=0.2, max_retries=0)
|
|
|
| def _get_council_llm_c() -> ChatGoogleGenerativeAI:
|
| """Return the Gemini model as the third council LLM."""
|
| return ChatGoogleGenerativeAI(model="gemini-2.5-flash", temperature=0.2, max_retries=0)
|
|
|
|
|
| def format_consensus_ui(label_a, label_b, label_c="", agreed=False, score=0.0, reason_a="", reason_b="", reason_c=""):
|
| """Generate an ultra-compact HTML Argument UI for 3 models."""
|
| status_icon = "✅ Match" if agreed else "⚠️ Diverge"
|
| status_color = "#2ecc71" if agreed else "#e67e22"
|
|
|
| return f"""
|
| <div style="margin-top:4px; border-left: 2px solid {status_color}; padding-left:8px; font-size:0.75rem;">
|
| <div style="color:{status_color}; font-weight:700; margin-bottom:2px;">{status_icon} (Max Match: {score})</div>
|
| <div style="display:flex; gap:10px;">
|
| <div style="flex:1; background:#0d1117; padding:6px; border-radius:4px; border:1px solid #30363d; color:#f0f6fc;">
|
| <b style="color:#7fb3f5; font-size:0.65rem;">MISTRAL:</b> {reason_a}
|
| </div>
|
| <div style="flex:1; background:#0d1117; padding:6px; border-radius:4px; border:1px solid #30363d; color:#f0f6fc;">
|
| <b style="color:#7fb3f5; font-size:0.65rem;">GROQ:</b> {reason_b}
|
| </div>
|
| <div style="flex:1; background:#0d1117; padding:6px; border-radius:4px; border:1px solid #30363d; color:#f0f6fc;">
|
| <b style="color:#7fb3f5; font-size:0.65rem;">GEMINI:</b> {reason_c}
|
| </div>
|
| </div>
|
| </div>
|
| """
|
|
|
| def _council_agreement_score(label_a: str, label_b: str) -> float:
|
| """Compute semantic cosine similarity between two label strings."""
|
| global _label_sim_model
|
| if _label_sim_model is None:
|
| _label_sim_model = SentenceTransformer("all-MiniLM-L6-v2")
|
|
|
| embs = _label_sim_model.encode([label_a, label_b])
|
| return round(float(cosine_similarity([embs[0]], [embs[1]])[0][0]), 3)
|
|
|
| |
