Spaces:
Runtime error
Runtime error
| """ | |
| app.py — BERTopic Thematic Analysis Agent · Gradio UI | |
| Braun & Clarke (2006) · 6-Phase Pipeline · Premium Research Interface | |
| """ | |
| import json | |
| import os | |
| import time | |
| import uuid | |
| from pathlib import Path | |
| import gradio as gr | |
| import pandas as pd | |
| # Lazy agent import to allow UI to load even if deps are missing | |
| def _get_agent_fn(): | |
| from agent import stream_agent_response | |
| return stream_agent_response | |
| DATA_DIR = Path("data") | |
| DATA_DIR.mkdir(exist_ok=True) | |
| THREAD_ID = str(uuid.uuid4()) | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| # PHASE METADATA | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| PHASES = [ | |
| {"id": 1, "name": "Familiarisation", "icon": "📖", "color": "#6366f1"}, | |
| {"id": 2, "name": "Initial Codes", "icon": "🏷️", "color": "#8b5cf6"}, | |
| {"id": 3, "name": "Themes", "icon": "🧩", "color": "#a855f7"}, | |
| {"id": 4, "name": "Saturation", "icon": "🔍", "color": "#d946ef"}, | |
| {"id": 5, "name": "Naming", "icon": "✏️", "color": "#ec4899"}, | |
| {"id": "5.5", "name": "PAJAIS", "icon": "🗂️", "color": "#f43f5e"}, | |
| {"id": 6, "name": "Report", "icon": "📄", "color": "#ef4444"}, | |
| ] | |
| CHART_OPTIONS = [ | |
| "Topic Size Distribution", | |
| "Inter-Topic Heatmap", | |
| "Coverage Curve", | |
| "PCA Topic Map", | |
| ] | |
| CHART_FILES = [ | |
| "chart_topic_sizes.html", | |
| "chart_heatmap.html", | |
| "chart_coverage.html", | |
| "chart_pca.html", | |
| ] | |
| # ── Review Table column config ────────────────────────────────────────────── | |
| REVIEW_COLS = ["#", "Topic Label", "Top Evidence", "Sentences", "Papers", "Approve", "Rename To", "Reasoning"] | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| # HTML / CSS COMPONENTS | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| GLOBAL_CSS = """ | |
| @import url('https://fonts.googleapis.com/css2?family=Syne:wght@400;500;600;700;800&family=JetBrains+Mono:wght@300;400;500&display=swap'); | |
| :root { | |
| --bg-void: #060608; | |
| --bg-surface: #0d0d12; | |
| --bg-elevated: #13131a; | |
| --bg-card: #1a1a24; | |
| --bg-hover: #20202e; | |
| --border: #ffffff0f; | |
| --border-glow: #6366f130; | |
| --accent-1: #6366f1; | |
| --accent-2: #8b5cf6; | |
| --accent-3: #a855f7; | |
| --accent-hot: #f472b6; | |
| --text-primary: #f0f0f8; | |
| --text-secondary:#9898b0; | |
| --text-muted: #505068; | |
| --success: #10b981; | |
| --warning: #f59e0b; | |
| --error: #ef4444; | |
| --font-display: 'Syne', sans-serif; | |
| --font-mono: 'JetBrains Mono', monospace; | |
| --radius-sm: 6px; | |
| --radius-md: 12px; | |
| --radius-lg: 18px; | |
| --radius-xl: 24px; | |
| --glow-purple: 0 0 40px rgba(99,102,241,0.15); | |
| --glow-pink: 0 0 40px rgba(244,114,182,0.12); | |
| } | |
| /* Reset & Base */ | |
| body, .gradio-container { | |
| background: var(--bg-void) !important; | |
| font-family: var(--font-display) !important; | |
| color: var(--text-primary) !important; | |
| } | |
| /* Scrollbar */ | |
| ::-webkit-scrollbar { width: 4px; height: 4px; } | |
| ::-webkit-scrollbar-track { background: var(--bg-surface); } | |
| ::-webkit-scrollbar-thumb { background: var(--accent-1); border-radius: 2px; } | |
| /* App Shell */ | |
| .gradio-container { max-width: 1400px !important; margin: 0 auto !important; padding: 0 !important; } | |
| footer { display: none !important; } | |
| /* ── Header ── */ | |
| #app-header { | |
| background: linear-gradient(135deg, var(--bg-surface) 0%, #0f0f18 100%); | |
| border-bottom: 1px solid var(--border); | |
| padding: 28px 40px 24px; | |
| position: relative; | |
| overflow: hidden; | |
| } | |
| #app-header::before { | |
| content: ''; | |
| position: absolute; | |
| top: -60px; left: -60px; | |
| width: 300px; height: 300px; | |
| background: radial-gradient(circle, rgba(99,102,241,0.12) 0%, transparent 70%); | |
| pointer-events: none; | |
| } | |
| #app-header::after { | |
| content: ''; | |
| position: absolute; | |
| bottom: -40px; right: 80px; | |
| width: 200px; height: 200px; | |
| background: radial-gradient(circle, rgba(168,85,247,0.08) 0%, transparent 70%); | |
| pointer-events: none; | |
| } | |
| .header-wordmark { | |
| font-size: 11px; | |
| letter-spacing: 4px; | |
| text-transform: uppercase; | |
| color: var(--accent-1); | |
| font-family: var(--font-mono); | |
| margin-bottom: 8px; | |
| opacity: 0.8; | |
| } | |
| .header-title { | |
| font-size: 32px; | |
| font-weight: 800; | |
| background: linear-gradient(135deg, #f0f0f8 0%, #a5a5c8 100%); | |
| -webkit-background-clip: text; | |
| -webkit-text-fill-color: transparent; | |
| background-clip: text; | |
| line-height: 1.1; | |
| margin: 0 0 8px; | |
| } | |
| .header-subtitle { | |
| font-size: 14px; | |
| color: var(--text-secondary); | |
| font-weight: 400; | |
| font-family: var(--font-mono); | |
| } | |
| .header-badges { | |
| display: flex; | |
| gap: 8px; | |
| margin-top: 16px; | |
| flex-wrap: wrap; | |
| } | |
| .badge { | |
| display: inline-flex; | |
| align-items: center; | |
| gap: 5px; | |
| padding: 4px 10px; | |
| border-radius: 20px; | |
| font-size: 11px; | |
| font-family: var(--font-mono); | |
| border: 1px solid; | |
| font-weight: 500; | |
| } | |
| .badge-purple { background: rgba(99,102,241,0.1); border-color: rgba(99,102,241,0.3); color: #a5b4fc; } | |
| .badge-violet { background: rgba(139,92,246,0.1); border-color: rgba(139,92,246,0.3); color: #c4b5fd; } | |
| .badge-pink { background: rgba(244,114,182,0.1); border-color: rgba(244,114,182,0.3); color: #f9a8d4; } | |
| .badge-green { background: rgba(16,185,129,0.1); border-color: rgba(16,185,129,0.3); color: #6ee7b7; } | |
| /* ── Phase Progress Bar ── */ | |
| #phase-bar-container { | |
| background: var(--bg-surface); | |
| border-bottom: 1px solid var(--border); | |
| padding: 16px 40px; | |
| } | |
| /* ── Section Panels ── */ | |
| .panel-card { | |
| background: var(--bg-card); | |
| border: 1px solid var(--border); | |
| border-radius: var(--radius-lg); | |
| overflow: hidden; | |
| margin-bottom: 0; | |
| transition: border-color 0.3s ease; | |
| } | |
| .panel-card:hover { border-color: var(--border-glow); } | |
| .panel-header { | |
| display: flex; | |
| align-items: center; | |
| gap: 10px; | |
| padding: 14px 20px; | |
| background: var(--bg-elevated); | |
| border-bottom: 1px solid var(--border); | |
| } | |
| .panel-icon { | |
| font-size: 18px; | |
| width: 32px; height: 32px; | |
| display: flex; align-items: center; justify-content: center; | |
| background: rgba(99,102,241,0.12); | |
| border-radius: 8px; | |
| } | |
| .panel-title { | |
| font-size: 13px; | |
| font-weight: 700; | |
| letter-spacing: 1.5px; | |
| text-transform: uppercase; | |
| color: var(--text-primary); | |
| } | |
| .panel-subtitle { | |
| font-size: 11px; | |
| color: var(--text-muted); | |
| font-family: var(--font-mono); | |
| margin-left: auto; | |
| } | |
| /* ── Gradio overrides ── */ | |
| .gr-button { | |
| font-family: var(--font-display) !important; | |
| font-weight: 600 !important; | |
| letter-spacing: 0.5px !important; | |
| border-radius: var(--radius-md) !important; | |
| transition: all 0.2s ease !important; | |
| } | |
| .gr-button-primary { | |
| background: linear-gradient(135deg, var(--accent-1), var(--accent-2)) !important; | |
| border: none !important; | |
| color: white !important; | |
| box-shadow: 0 4px 15px rgba(99,102,241,0.3) !important; | |
| } | |
| .gr-button-primary:hover { | |
| transform: translateY(-1px) !important; | |
| box-shadow: 0 6px 20px rgba(99,102,241,0.4) !important; | |
| } | |
| .gr-button-secondary { | |
| background: var(--bg-elevated) !important; | |
| border: 1px solid var(--border) !important; | |
| color: var(--text-primary) !important; | |
| } | |
| .gr-button-secondary:hover { | |
| background: var(--bg-hover) !important; | |
| border-color: var(--accent-1) !important; | |
| } | |
| /* Send button special */ | |
| #send-btn { | |
| background: linear-gradient(135deg, #6366f1, #8b5cf6) !important; | |
| min-width: 100px !important; | |
| } | |
| #submit-review-btn { | |
| background: linear-gradient(135deg, #10b981, #059669) !important; | |
| border: none !important; | |
| color: white !important; | |
| box-shadow: 0 4px 15px rgba(16,185,129,0.25) !important; | |
| } | |
| #submit-review-btn:hover { | |
| transform: translateY(-1px) !important; | |
| box-shadow: 0 6px 20px rgba(16,185,129,0.35) !important; | |
| } | |
| /* Chatbot */ | |
| .chatbot-container .message { | |
| border-radius: var(--radius-md) !important; | |
| } | |
| /* Input fields */ | |
| input, textarea { | |
| background: var(--bg-elevated) !important; | |
| border-color: var(--border) !important; | |
| color: var(--text-primary) !important; | |
| border-radius: var(--radius-md) !important; | |
| font-family: var(--font-display) !important; | |
| } | |
| input:focus, textarea:focus { | |
| border-color: var(--accent-1) !important; | |
| box-shadow: 0 0 0 3px rgba(99,102,241,0.15) !important; | |
| } | |
| /* Tabs */ | |
| .tab-nav button { | |
| font-family: var(--font-display) !important; | |
| font-weight: 600 !important; | |
| color: var(--text-secondary) !important; | |
| border-radius: var(--radius-sm) var(--radius-sm) 0 0 !important; | |
| transition: all 0.2s !important; | |
| } | |
| .tab-nav button.selected { | |
| color: var(--accent-1) !important; | |
| border-bottom: 2px solid var(--accent-1) !important; | |
| background: var(--bg-elevated) !important; | |
| } | |
| /* Dataframe */ | |
| .gr-dataframe table { | |
| font-family: var(--font-mono) !important; | |
| font-size: 12px !important; | |
| } | |
| .gr-dataframe th { | |
| background: var(--bg-elevated) !important; | |
| color: var(--accent-1) !important; | |
| font-size: 10px !important; | |
| letter-spacing: 1px !important; | |
| text-transform: uppercase !important; | |
| } | |
| .gr-dataframe td { border-color: var(--border) !important; } | |
| /* File upload zone */ | |
| .gr-file { | |
| border: 2px dashed var(--border-glow) !important; | |
| border-radius: var(--radius-lg) !important; | |
| background: var(--bg-elevated) !important; | |
| transition: all 0.3s !important; | |
| } | |
| .gr-file:hover { | |
| border-color: var(--accent-1) !important; | |
| background: rgba(99,102,241,0.04) !important; | |
| } | |
| /* Accordion */ | |
| .gr-accordion { border-color: var(--border) !important; } | |
| .gr-accordion-header { | |
| background: var(--bg-elevated) !important; | |
| color: var(--text-primary) !important; | |
| font-family: var(--font-display) !important; | |
| font-weight: 600 !important; | |
| } | |
| /* Stats strip */ | |
| .stats-strip { | |
| display: flex; | |
| gap: 1px; | |
| background: var(--border); | |
| border-radius: var(--radius-md); | |
| overflow: hidden; | |
| margin: 12px 0; | |
| } | |
| .stat-cell { | |
| flex: 1; | |
| padding: 14px 16px; | |
| background: var(--bg-elevated); | |
| text-align: center; | |
| } | |
| .stat-value { | |
| font-size: 22px; | |
| font-weight: 800; | |
| color: var(--text-primary); | |
| font-family: var(--font-mono); | |
| } | |
| .stat-label { | |
| font-size: 10px; | |
| color: var(--text-muted); | |
| text-transform: uppercase; | |
| letter-spacing: 1.5px; | |
| margin-top: 2px; | |
| } | |
| /* Section divider */ | |
| .section-divider { | |
| display: flex; | |
| align-items: center; | |
| gap: 12px; | |
| padding: 20px 40px; | |
| color: var(--text-muted); | |
| font-size: 11px; | |
| font-family: var(--font-mono); | |
| letter-spacing: 2px; | |
| text-transform: uppercase; | |
| } | |
| .section-divider::before, .section-divider::after { | |
| content: ''; | |
| flex: 1; | |
| height: 1px; | |
| background: var(--border); | |
| } | |
| /* Notification toast area */ | |
| .toast-area { | |
| position: fixed; | |
| top: 20px; | |
| right: 20px; | |
| z-index: 9999; | |
| display: flex; | |
| flex-direction: column; | |
| gap: 8px; | |
| pointer-events: none; | |
| } | |
| /* Download card */ | |
| .download-card { | |
| display: flex; | |
| align-items: center; | |
| gap: 12px; | |
| padding: 14px 16px; | |
| background: var(--bg-elevated); | |
| border: 1px solid var(--border); | |
| border-radius: var(--radius-md); | |
| margin-bottom: 8px; | |
| transition: all 0.2s; | |
| } | |
| .download-card:hover { | |
| border-color: var(--accent-1); | |
| background: var(--bg-hover); | |
| } | |
| .download-icon { font-size: 20px; } | |
| .download-info { flex: 1; } | |
| .download-name { font-size: 13px; font-weight: 600; } | |
| .download-desc { font-size: 11px; color: var(--text-secondary); font-family: var(--font-mono); } | |
| """ | |
| def make_phase_bar_html(current_phase: float = 0) -> str: | |
| """Render the 7-phase progress strip.""" | |
| phase_ids = [1, 2, 3, 4, 5, 5.5, 6] | |
| items = [] | |
| for i, p in enumerate(PHASES): | |
| pid = phase_ids[i] | |
| if pid < current_phase: | |
| state = "done" | |
| elif pid == current_phase: | |
| state = "active" | |
| else: | |
| state = "pending" | |
| state_styles = { | |
| "done": "background:rgba(16,185,129,0.15); border-color:rgba(16,185,129,0.4); color:#6ee7b7;", | |
| "active": f"background:rgba(99,102,241,0.2); border-color:{p['color']}; color:{p['color']}; box-shadow: 0 0 12px {p['color']}40;", | |
| "pending": "background:var(--bg-elevated); border-color:var(--border); color:var(--text-muted);", | |
| }[state] | |
| check = "✓ " if state == "done" else (f"{p['icon']} " if state == "active" else "") | |
| num_badge = f'<span style="font-size:9px;font-family:var(--font-mono);opacity:0.6;display:block;margin-bottom:1px">Ph {pid}</span>' | |
| items.append(f""" | |
| <div style="flex:1; padding:8px 6px; border:1px solid; border-radius:8px; | |
| text-align:center; font-size:11px; font-weight:700; | |
| transition:all 0.3s; cursor:default; {state_styles}"> | |
| {num_badge}{check}{p['name']} | |
| </div>""") | |
| return f""" | |
| <div style="padding:14px 40px; background:var(--bg-surface); border-bottom:1px solid var(--border);"> | |
| <div style="font-size:10px; color:var(--text-muted); font-family:var(--font-mono); | |
| letter-spacing:2px; text-transform:uppercase; margin-bottom:10px;"> | |
| ── Braun & Clarke (2006) · Analysis Progress | |
| </div> | |
| <div style="display:flex; gap:6px; align-items:stretch;"> | |
| {''.join(items)} | |
| </div> | |
| </div> | |
| """ | |
| def make_header_html() -> str: | |
| return """ | |
| <div id="app-header"> | |
| <div class="header-wordmark">Computational Thematic Analysis System</div> | |
| <div class="header-title">BERTopic Agent</div> | |
| <div class="header-subtitle">all-MiniLM-L6-v2 · AgglomerativeClustering · Mistral Large · LangGraph ReAct</div> | |
| <div class="header-badges"> | |
| <span class="badge badge-purple">📐 Braun & Clarke (2006)</span> | |
| <span class="badge badge-violet">🔬 BERTopic Discovery</span> | |
| <span class="badge badge-pink">🗂️ PAJAIS 25 Taxonomy</span> | |
| <span class="badge badge-green">✓ 7 Analysis Phases</span> | |
| </div> | |
| </div> | |
| """ | |
| def make_chart_iframe(chart_path: str) -> str: | |
| full_path = DATA_DIR / chart_path | |
| if not full_path.exists(): | |
| return """ | |
| <div style="height:460px; display:flex; align-items:center; justify-content:center; | |
| background:var(--bg-elevated); border-radius:var(--radius-lg); | |
| border:1px dashed var(--border); flex-direction:column; gap:12px;"> | |
| <div style="font-size:32px;">📊</div> | |
| <div style="color:var(--text-muted); font-family:var(--font-mono); font-size:13px;"> | |
| Chart available after Phase 2 discovery | |
| </div> | |
| </div> | |
| """ | |
| html_content = full_path.read_text() | |
| # Encode for iframe srcDoc | |
| import base64 | |
| encoded = base64.b64encode(html_content.encode()).decode() | |
| return f'<iframe src="data:text/html;base64,{encoded}" width="100%" height="480px" style="border:none; border-radius:var(--radius-lg);"></iframe>' | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| # STATE HELPERS | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| def _load_app_state() -> dict: | |
| p = DATA_DIR / "state.json" | |
| return json.loads(p.read_text()) if p.exists() else {"phase": 0} | |
| def _load_summaries_as_table(): | |
| p = DATA_DIR / "summaries.json" | |
| if not p.exists(): | |
| return pd.DataFrame(columns=REVIEW_COLS) | |
| summaries = json.load(open(p)) | |
| rows = list(map(lambda s: [ | |
| s.get("display_id", s["topic_id"]), | |
| s.get("label", f"Topic_{s['topic_id']}"), | |
| s.get("top_sentences", [""])[0][:120] if s.get("top_sentences") else "", | |
| len(s.get("top_sentences", [])), | |
| s.get("size", 0), | |
| s.get("approved", False), | |
| s.get("rename_to", ""), | |
| s.get("reasoning", ""), | |
| ], summaries)) | |
| return pd.DataFrame(rows, columns=REVIEW_COLS) | |
| def _download_files_list(): | |
| patterns = ["comparison.csv", "narrative.md", "narrative.txt", | |
| "summaries.json", "themes.json", "taxonomy_mapping.json", | |
| "chart_topic_sizes.html", "chart_heatmap.html", | |
| "chart_coverage.html", "chart_pca.html"] | |
| return list(filter(None, [str(DATA_DIR / p) if (DATA_DIR / p).exists() else None for p in patterns])) | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| # EVENT HANDLERS | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| def handle_file_upload(file_obj, run_mode_radio): | |
| if file_obj is None: | |
| return gr.update(), "No file selected.", gr.update() | |
| path = file_obj.name | |
| # Quick preview | |
| df = pd.read_csv(path, nrows=5) | |
| stats_html = f""" | |
| <div class="stats-strip"> | |
| <div class="stat-cell"><div class="stat-value">{len(df.columns)}</div><div class="stat-label">Columns</div></div> | |
| <div class="stat-cell"><div class="stat-value">CSV</div><div class="stat-label">Format</div></div> | |
| <div class="stat-cell"><div class="stat-value">{run_mode_radio.upper()}</div><div class="stat-label">Run Mode</div></div> | |
| </div> | |
| <div style="font-family:var(--font-mono); font-size:11px; color:var(--text-muted); padding:4px 0;"> | |
| Columns detected: {', '.join(df.columns[:8].tolist())} | |
| </div> | |
| """ | |
| ready_msg = f"✓ File loaded: `{Path(path).name}` · {run_mode_radio} mode selected. Send a message to start Phase 1." | |
| return gr.update(value=stats_html), ready_msg, gr.update(value=path) | |
| def handle_send(message, history, csv_path_state, run_mode_radio): | |
| if not message.strip(): | |
| return history, "", gr.update(), gr.update(), gr.update() | |
| # Inject CSV path context if available | |
| enriched = message | |
| if csv_path_state and "phase 1" in message.lower() or (csv_path_state and not any("[CSV:" in m[0] for m in history)): | |
| enriched = f"{message}\n\n[CSV path for tool: {csv_path_state}] [Run mode: {run_mode_radio}]" | |
| # Add user message | |
| history = history + [[message, None]] | |
| yield history, "", gr.update(), gr.update(), gr.update() | |
| # Get agent response | |
| try: | |
| stream_fn = _get_agent_fn() | |
| response = stream_fn(enriched, history, THREAD_ID) | |
| except Exception as e: | |
| response = f"⚠️ Agent error: {str(e)}\n\nPlease check your MISTRAL_API_KEY environment variable." | |
| history[-1][1] = response | |
| # Update phase bar and table | |
| state = _load_app_state() | |
| phase = state.get("phase", 0) | |
| table_df = _load_summaries_as_table() | |
| phase_html = make_phase_bar_html(phase) | |
| yield history, "", gr.update(value=phase_html), gr.update(value=table_df), gr.update() | |
| def handle_submit_review(table_data, history): | |
| if table_data is None or len(table_data) == 0: | |
| return history, gr.update() | |
| # Convert dataframe to review_json | |
| df = table_data if isinstance(table_data, pd.DataFrame) else pd.DataFrame(table_data, columns=REVIEW_COLS) | |
| # Load summaries to map display_id → topic_id | |
| summaries_path = DATA_DIR / "summaries.json" | |
| if summaries_path.exists(): | |
| summaries = json.load(open(summaries_path)) | |
| id_map = {s.get("display_id", s["topic_id"]): s["topic_id"] for s in summaries} | |
| else: | |
| id_map = {} | |
| def _row_to_dict(row): | |
| display_id = row["#"] if "#" in row else row.iloc[0] | |
| topic_id = id_map.get(int(display_id) if str(display_id).isdigit() else display_id, display_id) | |
| approve_val = row["Approve"] if "Approve" in row else row.iloc[5] | |
| return { | |
| "topic_id": topic_id, | |
| "approved": bool(approve_val) if not isinstance(approve_val, str) else approve_val.lower() in ("true", "1", "yes", "✓"), | |
| "rename_to": str(row["Rename To"] if "Rename To" in row else row.iloc[6]).strip(), | |
| "reasoning": str(row["Reasoning"] if "Reasoning" in row else row.iloc[7]).strip(), | |
| } | |
| review_rows = list(map(_row_to_dict, [df.iloc[i] for i in range(len(df))])) | |
| review_json = json.dumps(review_rows) | |
| # Send to agent | |
| submit_msg = f"[REVIEW SUBMITTED] The researcher has completed their review of {len(review_rows)} topics. Approved: {sum(1 for r in review_rows if r['approved'])}. Review data: {review_json[:500]}..." | |
| full_msg = f"Submit Review table received. Please proceed with consolidate_into_themes using this review data: {review_json}" | |
| history = history + [[None, None]] | |
| try: | |
| stream_fn = _get_agent_fn() | |
| response = stream_fn(full_msg, history, THREAD_ID) | |
| except Exception as e: | |
| response = f"⚠️ Error processing review: {str(e)}" | |
| history[-1] = [f"📋 Submitted review of {sum(1 for r in review_rows if r['approved'])} approved topics", response] | |
| state = _load_app_state() | |
| phase_html = make_phase_bar_html(state.get("phase", 0)) | |
| return history, gr.update(value=phase_html) | |
| def handle_chart_select(chart_name): | |
| idx = CHART_OPTIONS.index(chart_name) if chart_name in CHART_OPTIONS else 0 | |
| return make_chart_iframe(CHART_FILES[idx]) | |
| def refresh_downloads(): | |
| files = _download_files_list() | |
| return gr.update(value=files if files else None) | |
| def make_download_gallery_html(): | |
| files = _download_files_list() | |
| if not files: | |
| return """ | |
| <div style="text-align:center; padding:40px; color:var(--text-muted); font-family:var(--font-mono); font-size:13px;"> | |
| 📦 Output files will appear here after analysis phases complete | |
| </div>""" | |
| icons = {"csv": "📊", "json": "🗃️", "md": "📝", "txt": "📄", "html": "📈", "npy": "🔢"} | |
| def _card(fp): | |
| p = Path(fp) | |
| ext = p.suffix.lstrip(".") | |
| icon = icons.get(ext, "📁") | |
| size = p.stat().st_size | |
| size_str = f"{size/1024:.1f} KB" if size < 1024*1024 else f"{size/1024/1024:.1f} MB" | |
| return f""" | |
| <div class="download-card"> | |
| <span class="download-icon">{icon}</span> | |
| <div class="download-info"> | |
| <div class="download-name">{p.name}</div> | |
| <div class="download-desc">{ext.upper()} · {size_str}</div> | |
| </div> | |
| </div>""" | |
| return "".join(map(_card, files)) | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| # BUILD UI | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| with gr.Blocks(title="BERTopic Thematic Analysis Agent") as demo: | |
| # ── State ──────────────────────────────────────────────────────────────── | |
| csv_path_state = gr.State(value="") | |
| # ── Header ─────────────────────────────────────────────────────────────── | |
| gr.HTML(make_header_html()) | |
| # ── Phase Progress Bar ─────────────────────────────────────────────────── | |
| phase_bar = gr.HTML(make_phase_bar_html(0), elem_id="phase-bar") | |
| # ── Main Layout: Left Column + Right Column ────────────────────────────── | |
| with gr.Row(equal_height=False): | |
| # ════════════════════════════════════════════════════ | |
| # LEFT COLUMN — Data Input + Conversation | |
| # ════════════════════════════════════════════════════ | |
| with gr.Column(scale=4, min_width=480): | |
| # ── Section 1: Data Input ──────────────────────── | |
| gr.HTML(""" | |
| <div style="padding:16px 0 8px;"> | |
| <div class="panel-header" style="border-radius:12px 12px 0 0; border:1px solid var(--border);"> | |
| <div class="panel-icon">📂</div> | |
| <div> | |
| <div class="panel-title">Data Input</div> | |
| <div style="font-size:11px;color:var(--text-muted);font-family:var(--font-mono);">Scopus CSV · Phase 1 Familiarisation</div> | |
| </div> | |
| </div> | |
| </div> | |
| """) | |
| with gr.Group(): | |
| with gr.Row(): | |
| with gr.Column(scale=3): | |
| file_input = gr.File( | |
| label="Upload Scopus Export (.csv)", | |
| file_types=[".csv"], | |
| elem_id="csv-upload", | |
| ) | |
| with gr.Column(scale=2): | |
| run_mode_radio = gr.Radio( | |
| choices=["abstract", "title"], | |
| value="abstract", | |
| label="Clustering Mode", | |
| info="Which field to embed & cluster", | |
| ) | |
| file_stats_html = gr.HTML( | |
| value="""<div style="color:var(--text-muted);font-family:var(--font-mono);font-size:12px;padding:8px 0;"> | |
| ↑ Upload a Scopus CSV to begin. Required columns: Title, Abstract.</div>""" | |
| ) | |
| # ── Section 2: Agent Conversation ─────────────────── | |
| gr.HTML(""" | |
| <div style="padding:16px 0 8px;"> | |
| <div class="panel-header" style="border-radius:12px 12px 0 0; border:1px solid var(--border);"> | |
| <div class="panel-icon">🤖</div> | |
| <div> | |
| <div class="panel-title">Agent Conversation</div> | |
| <div style="font-size:11px;color:var(--text-muted);font-family:var(--font-mono);">LangGraph ReAct · Mistral Large · MemorySaver</div> | |
| </div> | |
| <div style="margin-left:auto; display:flex; gap:6px; align-items:center;"> | |
| <div style="width:6px;height:6px;border-radius:50%;background:#10b981;box-shadow:0 0 6px #10b981;"></div> | |
| <span style="font-size:11px;color:#10b981;font-family:var(--font-mono);">AGENT READY</span> | |
| </div> | |
| </div> | |
| </div> | |
| """) | |
| chatbot = gr.Chatbot( | |
| value=[], | |
| label="", | |
| height=520, | |
| elem_id="main-chatbot", | |
| avatar_images=( | |
| None, # user | |
| "https://api.dicebear.com/7.x/bottts/svg?seed=bertopic&backgroundColor=6366f1", | |
| ), | |
| render_markdown=True, | |
| ) | |
| with gr.Row(): | |
| chat_input = gr.Textbox( | |
| placeholder='e.g. "Start Phase 1 with the uploaded file" or "Proceed to Phase 2"', | |
| label="", | |
| lines=2, | |
| max_lines=5, | |
| show_label=False, | |
| scale=5, | |
| elem_id="chat-input", | |
| ) | |
| with gr.Column(scale=1, min_width=110): | |
| send_btn = gr.Button("Send ↑", variant="primary", elem_id="send-btn", size="lg") | |
| clear_btn = gr.Button("Clear", variant="secondary", size="sm") | |
| # Quick-start prompts | |
| gr.HTML(""" | |
| <div style="display:flex; gap:6px; flex-wrap:wrap; padding:8px 0 0;"> | |
| <div style="font-size:10px;color:var(--text-muted);font-family:var(--font-mono); | |
| display:flex;align-items:center;margin-right:4px;">QUICK START →</div> | |
| </div> | |
| """) | |
| with gr.Row(): | |
| q1 = gr.Button("📖 Start Phase 1", size="sm", variant="secondary") | |
| q2 = gr.Button("🏷️ Run Phase 2", size="sm", variant="secondary") | |
| q3 = gr.Button("🧩 Proceed Phase 3", size="sm", variant="secondary") | |
| q4 = gr.Button("📄 Generate Report", size="sm", variant="secondary") | |
| # ════════════════════════════════════════════════════ | |
| # RIGHT COLUMN — Results Tabs | |
| # ════════════════════════════════════════════════════ | |
| with gr.Column(scale=6, min_width=600): | |
| gr.HTML(""" | |
| <div style="padding:16px 0 8px;"> | |
| <div class="panel-header" style="border-radius:12px 12px 0 0; border:1px solid var(--border);"> | |
| <div class="panel-icon">📊</div> | |
| <div> | |
| <div class="panel-title">Analysis Results</div> | |
| <div style="font-size:11px;color:var(--text-muted);font-family:var(--font-mono);">Review · Charts · Downloads</div> | |
| </div> | |
| </div> | |
| </div> | |
| """) | |
| with gr.Tabs(elem_id="results-tabs"): | |
| # ─── Tab 1: Review Table ───────────────────────── | |
| with gr.Tab("🗃️ Review Table", id="tab-review"): | |
| gr.HTML(""" | |
| <div style="padding:12px 0 4px; color:var(--text-secondary); font-size:12px; font-family:var(--font-mono); line-height:1.6;"> | |
| ✏️ Edit cells directly · Tick <strong style="color:#10b981">Approve</strong> for topics to include · | |
| Group related topics with the same <strong style="color:#a5b4fc">Rename To</strong> label · | |
| Click <strong style="color:#10b981">Submit Review</strong> when done | |
| </div> | |
| """) | |
| review_table = gr.Dataframe( | |
| value=_load_summaries_as_table(), | |
| headers=REVIEW_COLS, | |
| datatype=["number", "str", "str", "number", "number", "bool", "str", "str"], | |
| column_count=(8, "fixed"), | |
| interactive=True, | |
| wrap=True, | |
| max_height=520, | |
| elem_id="review-table", | |
| column_widths=["40px", "160px", "260px", "70px", "60px", "65px", "130px", "200px"], | |
| ) | |
| with gr.Row(): | |
| submit_review_btn = gr.Button( | |
| "✓ Submit Review → Phase Advance", | |
| variant="primary", | |
| elem_id="submit-review-btn", | |
| scale=2, | |
| ) | |
| refresh_table_btn = gr.Button("⟳ Refresh", variant="secondary", scale=1) | |
| approve_all_btn = gr.Button("☑ Approve All", variant="secondary", scale=1) | |
| with gr.Accordion("📋 Review Instructions", open=False): | |
| gr.Markdown(""" | |
| **Column Guide:** | |
| - **#** — Topic display ID (auto-assigned, do not edit) | |
| - **Topic Label** — LLM-generated label (editable) | |
| - **Top Evidence** — Most representative sentence for this topic | |
| - **Sentences** — Count of representative sentences loaded | |
| - **Papers** — Number of papers in this cluster | |
| - **Approve** ✓ — Check to include this topic in the thematic analysis | |
| - **Rename To** — Enter a theme name to group multiple topics (same name = same theme) | |
| - **Reasoning** — Document your qualitative decision for audit trail | |
| **Grouping Workflow:** | |
| 1. Review topics with similar content | |
| 2. Give them the same **Rename To** label (e.g., "AI Ethics") | |
| 3. All approved topics with the same rename become one consolidated theme | |
| 4. Topics without a rename keep their original label | |
| **Stop Gates:** | |
| - After Phase 2: Review & approve initial topic codes | |
| - After Phase 3: Review consolidated themes | |
| - After Phase 4: Confirm saturation & coverage | |
| - After Phase 5.5: Validate PAJAIS taxonomy mappings | |
| """) | |
| # ─── Tab 2: Charts ─────────────────────────────── | |
| with gr.Tab("📈 Charts", id="tab-charts"): | |
| with gr.Row(): | |
| chart_dropdown = gr.Dropdown( | |
| choices=CHART_OPTIONS, | |
| value=CHART_OPTIONS[0], | |
| label="Select Chart", | |
| scale=3, | |
| interactive=True, | |
| ) | |
| refresh_charts_btn = gr.Button("⟳ Refresh Charts", variant="secondary", scale=1) | |
| chart_display = gr.HTML( | |
| value=make_chart_iframe(CHART_FILES[0]), | |
| elem_id="chart-frame", | |
| ) | |
| gr.HTML(""" | |
| <div style="display:grid; grid-template-columns:1fr 1fr; gap:8px; margin-top:12px;"> | |
| <div style="padding:12px; background:var(--bg-elevated); border-radius:8px; border:1px solid var(--border);"> | |
| <div style="font-size:11px; color:var(--accent-1); font-family:var(--font-mono); font-weight:700; margin-bottom:4px;">📊 TOPIC SIZES</div> | |
| <div style="font-size:11px; color:var(--text-secondary);">Bar chart of top 40 topics ranked by sentence count</div> | |
| </div> | |
| <div style="padding:12px; background:var(--bg-elevated); border-radius:8px; border:1px solid var(--border);"> | |
| <div style="font-size:11px; color:var(--accent-2); font-family:var(--font-mono); font-weight:700; margin-bottom:4px;">🔥 SIMILARITY HEATMAP</div> | |
| <div style="font-size:11px; color:var(--text-secondary);">Cosine similarity matrix of top 20 topic centroids</div> | |
| </div> | |
| <div style="padding:12px; background:var(--bg-elevated); border-radius:8px; border:1px solid var(--border);"> | |
| <div style="font-size:11px; color:var(--accent-3); font-family:var(--font-mono); font-weight:700; margin-bottom:4px;">📈 COVERAGE CURVE</div> | |
| <div style="font-size:11px; color:var(--text-secondary);">Cumulative sentence coverage as topics are added</div> | |
| </div> | |
| <div style="padding:12px; background:var(--bg-elevated); border-radius:8px; border:1px solid var(--border);"> | |
| <div style="font-size:11px; color:var(--accent-hot); font-family:var(--font-mono); font-weight:700; margin-bottom:4px;">🗺️ PCA TOPIC MAP</div> | |
| <div style="font-size:11px; color:var(--text-secondary);">2-D centroid projection coloured by cluster size</div> | |
| </div> | |
| </div> | |
| """) | |
| # ─── Tab 3: Download ───────────────────────────── | |
| with gr.Tab("⬇️ Download", id="tab-download"): | |
| gr.HTML(""" | |
| <div style="padding:12px 0 16px; color:var(--text-secondary); font-size:12px; font-family:var(--font-mono);"> | |
| All output files generated during the analysis. Files appear as each phase completes. | |
| </div> | |
| """) | |
| download_gallery_html = gr.HTML( | |
| value=make_download_gallery_html(), | |
| elem_id="download-gallery", | |
| ) | |
| download_file_widget = gr.File( | |
| label="Download Files", | |
| file_count="multiple", | |
| interactive=False, | |
| value=_download_files_list() or None, | |
| ) | |
| with gr.Row(): | |
| refresh_downloads_btn = gr.Button("⟳ Refresh Downloads", variant="secondary", scale=2) | |
| gr.HTML(""" | |
| <div style="font-size:11px; color:var(--text-muted); font-family:var(--font-mono); | |
| display:flex; align-items:center; padding:0 12px;"> | |
| Files auto-generate as analysis progresses through phases | |
| </div> | |
| """) | |
| gr.HTML(""" | |
| <div style="margin-top:16px; padding:14px 16px; background:var(--bg-elevated); | |
| border-radius:10px; border:1px solid var(--border-glow);"> | |
| <div style="font-size:11px; color:var(--accent-1); font-family:var(--font-mono); | |
| font-weight:700; margin-bottom:8px;">📦 EXPECTED OUTPUT FILES</div> | |
| <div style="display:grid; grid-template-columns:1fr 1fr; gap:4px;"> | |
| <div style="font-size:11px; color:var(--text-secondary); font-family:var(--font-mono);">📊 comparison.csv — Abstract vs Title themes</div> | |
| <div style="font-size:11px; color:var(--text-secondary); font-family:var(--font-mono);">📝 narrative.md — Section 7 Discussion</div> | |
| <div style="font-size:11px; color:var(--text-secondary); font-family:var(--font-mono);">🗃️ summaries.json — All topic data</div> | |
| <div style="font-size:11px; color:var(--text-secondary); font-family:var(--font-mono);">🗃️ themes.json — Consolidated themes</div> | |
| <div style="font-size:11px; color:var(--text-secondary); font-family:var(--font-mono);">🗃️ taxonomy_mapping.json — PAJAIS map</div> | |
| <div style="font-size:11px; color:var(--text-secondary); font-family:var(--font-mono);">📈 4x Plotly HTML charts</div> | |
| </div> | |
| </div> | |
| """) | |
| # ── Footer ─────────────────────────────────────────────────────────────── | |
| gr.HTML(""" | |
| <div style="text-align:center; padding:24px 40px 32px; | |
| border-top:1px solid var(--border); margin-top:24px; | |
| color:var(--text-muted); font-family:var(--font-mono); font-size:11px;"> | |
| <div style="margin-bottom:8px;"> | |
| BERTopic Thematic Analysis Agent · Braun & Clarke (2006) · PAJAIS 25-Category Taxonomy | |
| </div> | |
| <div style="display:flex; justify-content:center; gap:24px; flex-wrap:wrap;"> | |
| <span>all-MiniLM-L6-v2 embeddings</span> | |
| <span>·</span> | |
| <span>AgglomerativeClustering (cosine, threshold=0.7)</span> | |
| <span>·</span> | |
| <span>Mistral Large LLM</span> | |
| <span>·</span> | |
| <span>LangGraph ReAct + MemorySaver</span> | |
| </div> | |
| </div> | |
| """) | |
| # ══════════════════════════════════════════════════════════════════════════ | |
| # EVENT WIRING | |
| # ══════════════════════════════════════════════════════════════════════════ | |
| # File upload | |
| file_input.change( | |
| fn=handle_file_upload, | |
| inputs=[file_input, run_mode_radio], | |
| outputs=[file_stats_html, chat_input, csv_path_state], | |
| ) | |
| # Send message | |
| send_outputs = [chatbot, chat_input, phase_bar, review_table, download_gallery_html] | |
| send_btn.click( | |
| fn=handle_send, | |
| inputs=[chat_input, chatbot, csv_path_state, run_mode_radio], | |
| outputs=send_outputs, | |
| ) | |
| chat_input.submit( | |
| fn=handle_send, | |
| inputs=[chat_input, chatbot, csv_path_state, run_mode_radio], | |
| outputs=send_outputs, | |
| ) | |
| # Clear | |
| clear_btn.click(fn=lambda: ([], ""), outputs=[chatbot, chat_input]) | |
| # Quick prompts | |
| q1.click(fn=lambda: "Please start Phase 1. Load and familiarise with the uploaded Scopus CSV.", outputs=chat_input) | |
| q2.click(fn=lambda: "Phase 1 confirmed. Please proceed to Phase 2: run BERTopic discovery and generate topic labels.", outputs=chat_input) | |
| q3.click(fn=lambda: "Review submitted. Please proceed to Phase 3 and consolidate the approved topics into themes.", outputs=chat_input) | |
| q4.click(fn=lambda: "Themes confirmed and named. Please proceed to Phase 5.5 PAJAIS mapping, then Phase 6 to generate the report.", outputs=chat_input) | |
| # Submit Review | |
| submit_review_btn.click( | |
| fn=handle_submit_review, | |
| inputs=[review_table, chatbot], | |
| outputs=[chatbot, phase_bar], | |
| ) | |
| # Refresh table | |
| refresh_table_btn.click( | |
| fn=lambda: gr.update(value=_load_summaries_as_table()), | |
| outputs=review_table, | |
| ) | |
| # Approve all | |
| def _approve_all(df): | |
| if df is not None and len(df) > 0: | |
| result = df.copy() if isinstance(df, pd.DataFrame) else pd.DataFrame(df, columns=REVIEW_COLS) | |
| result["Approve"] = True | |
| return gr.update(value=result) | |
| return gr.update() | |
| approve_all_btn.click(fn=_approve_all, inputs=review_table, outputs=review_table) | |
| # Chart selection | |
| chart_dropdown.change( | |
| fn=handle_chart_select, | |
| inputs=chart_dropdown, | |
| outputs=chart_display, | |
| ) | |
| refresh_charts_btn.click( | |
| fn=lambda c: make_chart_iframe(CHART_FILES[CHART_OPTIONS.index(c) if c in CHART_OPTIONS else 0]), | |
| inputs=chart_dropdown, | |
| outputs=chart_display, | |
| ) | |
| # Downloads | |
| refresh_downloads_btn.click( | |
| fn=lambda: (refresh_downloads(), make_download_gallery_html()), | |
| outputs=[download_file_widget, download_gallery_html], | |
| ) | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| # LAUNCH | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| if __name__ == "__main__": | |
| demo.launch( | |
| server_name="0.0.0.0", | |
| server_port=7860, | |
| share=False, | |
| css=GLOBAL_CSS, | |
| theme=gr.themes.Base( | |
| primary_hue="indigo", | |
| neutral_hue="slate", | |
| font=gr.themes.GoogleFont("Syne"), | |
| ), | |
| ) |