app.py

"""
app.py
------
Gradio UI for the Journal Topic Modelling Agent.
Deploy on Hugging Face Spaces — add MISTRAL_API_KEY as a Space Secret.

Fixes applied vs original:
  • demo.queue() added  (required for generator streaming)
  • Removed gr.State() from .click() outputs (invalid in Gradio 4+)
  • clear_btn outputs match run_btn outputs exactly (10 components)
  • error display uses visible Markdown, not a hidden component
  • run_pipeline yields exactly 10 values matching 10 output components
  • mistralai import fixed in agent.py (from mistralai.client import Mistral)
"""

import gradio as gr
import json
import os
import traceback
import time
import threading
from queue import Queue, Empty

try:
    from agent import run_topic_modelling_agent
    AGENT_OK = True
    AGENT_ERR = ""
except Exception as _e:
    AGENT_OK = False
    AGENT_ERR = str(_e)

# ── Sample data ───────────────────────────────────────────────────────────────

SAMPLE_CSV = """title,abstract,year,journal
Deep Learning for Medical Imaging,This paper presents a convolutional neural network approach for automated diagnosis from MRI scans using deep learning techniques and transfer learning.,2022,PAJAIS
Blockchain Privacy in Healthcare,We examine privacy-preserving mechanisms in blockchain-based electronic health record systems and propose a new consensus protocol.,2023,PAJAIS
Explainable AI in Decision Support,An explainable artificial intelligence framework for clinical decision support systems is proposed and evaluated on hospital datasets.,2022,MIS Quarterly
Social Media Sentiment Analysis,This study analyses sentiment patterns in social media data using transformer-based NLP models with attention mechanisms.,2023,PAJAIS
Knowledge Graph Construction,Automated knowledge graph construction from unstructured text using entity recognition and relation extraction pipelines.,2021,PAJAIS
Federated Learning and Privacy,Federated machine learning enables model training across distributed clients without sharing raw data thereby preserving user privacy.,2023,JAIS
Chatbot Design for Customer Service,Conversational AI and chatbot design principles for improving customer service interactions and reducing wait times in banking.,2022,PAJAIS
IoT Security Vulnerabilities,An empirical study of security vulnerabilities in Internet of Things devices deployed in smart home and industrial environments.,2021,PAJAIS
Recommender Systems Bias,Investigating algorithmic bias and fairness issues in collaborative filtering recommender systems with mitigation strategies.,2023,PAJAIS
Digital Transformation SME,Digital transformation challenges and critical success factors for small and medium enterprises in emerging Asian markets.,2022,PAJAIS
Cloud ERP Adoption,Factors influencing cloud-based ERP system adoption in manufacturing firms using the technology-organisation-environment framework.,2021,PAJAIS
NLP Text Summarisation,Automatic text summarisation using pre-trained large language models for scientific document processing and literature review.,2023,PAJAIS
Cybersecurity Threat Detection,Machine learning models for real-time cybersecurity threat detection and anomaly identification in enterprise network traffic.,2022,PAJAIS
Human-Robot Interaction,User experience evaluation of social robots in elderly care settings using mixed methods with qualitative interviews.,2021,PAJAIS
Predictive Analytics Supply Chain,Predictive analytics and demand forecasting in supply chain management using gradient boosting and time series models.,2023,PAJAIS
"""

# ── Helpers ───────────────────────────────────────────────────────────────────

def read_csv_from_file(filepath: str) -> str:
    if not filepath:
        return ""
    try:
        with open(filepath, "r", encoding="utf-8", errors="replace") as f:
            return f.read()
    except Exception as e:
        return f"FILE_READ_ERROR: {e}"


def make_topics_markdown(frequency_table: list) -> str:
    if not frequency_table:
        return "_No topics found._"
    lines = [
        "| # | Topic | Label | Frequency | % Docs |",
        "|---|-------|-------|:---------:|:------:|",
    ]
    for i, item in enumerate(frequency_table, 1):
        lines.append(
            f"| {i} | {item['topic']} | {item['label']} "
            f"| {item['frequency']} | {item['percentage']}% |"
        )
    return "\n".join(lines)


def make_comparison_markdown(comparison: list) -> str:
    if not comparison:
        return "_No comparison data._"
    lines = [
        "| Topic | Title % | Abstract % | Δ (Abstract − Title) |",
        "|-------|:-------:|:----------:|:--------------------:|",
    ]
    for item in comparison[:40]:
        d = item["delta_pct"]
        arrow = "▲" if d > 0 else ("▼" if d < 0 else "–")
        lines.append(
            f"| {item['topic']} | {item['title_pct']}% "
            f"| {item['abstract_pct']}% | {arrow} {abs(d)}% |"
        )
    return "\n".join(lines)


def make_pajais_markdown(taxonomy_result: dict) -> str:
    if not taxonomy_result:
        return "_No taxonomy data._"
    cov = taxonomy_result.get("coverage_pct", 0)
    mapped_n = taxonomy_result.get("mapped_count", 0)
    novel_n = taxonomy_result.get("novel_count", 0)
    lines = [
        f"**PAJAIS Coverage:** {cov}%  &nbsp;|&nbsp;  "
        f"**Mapped:** {mapped_n}  &nbsp;|&nbsp;  **NOVEL:** {novel_n}\n",
        "---",
        "### 🗺️ MAPPED to PAJAIS Themes\n",
    ]
    for item in taxonomy_result.get("mapped", []):
        lines.append(
            f"- **{item['pajais_theme']}** → `{item['topic']}` — *{item['label']}*"
        )
    lines += ["\n---", "### 🆕 NOVEL Themes (no PAJAIS match)\n"]
    for item in taxonomy_result.get("novel", []):
        lines.append(f"- `{item['topic']}` — *{item['label']}*")
    return "\n".join(lines)


def make_summary_markdown(summary: dict) -> str:
    return (
        f"| Metric | Value |\n"
        f"|--------|-------|\n"
        f"| 📄 Papers analysed | **{summary['papers_analysed']}** |\n"
        f"| 🏷️ Topics discovered | **{summary['topics_discovered']}** |\n"
        f"| 🗺️ PAJAIS-mapped | **{summary['pajais_mapped']}** |\n"
        f"| 🆕 NOVEL themes | **{summary['novel_themes']}** |\n"
        f"| 📊 PAJAIS coverage | **{summary['pajais_coverage_pct']}%** |"
    )


# ── Pipeline generator ────────────────────────────────────────────────────────
# Yields exactly 10 values matching the 10 output components:
#   log_box, topics_md, comparison_md, pajais_md,
#   narrative_txt, dl_csv, dl_json, dl_txt, summary_md, err_md

EMPTY_10 = ("", "", "", "", "", None, None, None, "", "")


def run_pipeline(csv_file, csv_text_input, use_sample):
    # ── Determine CSV source ──
    if use_sample:
        csv_text = SAMPLE_CSV
    elif csv_file:
        csv_text = read_csv_from_file(csv_file)
        if csv_text.startswith("FILE_READ_ERROR"):
            yield ("", "", "", "", "", None, None, None, "", f"❌ {csv_text}")
            return
    elif csv_text_input and csv_text_input.strip():
        csv_text = csv_text_input.strip()
    else:
        yield (
            "",  "", "", "", "", None, None, None, "",
            "❌ Please upload a CSV file, paste CSV text, or tick **Use sample data**.",
        )
        return

    # ── Check API key ──
    if not os.environ.get("MISTRAL_API_KEY", "").strip():
        yield (
            "", "", "", "", "", None, None, None, "",
            "❌ **MISTRAL_API_KEY** is not set.\n\n"
            "Go to your Hugging Face Space → **Settings → Variables and Secrets** "
            "and add `MISTRAL_API_KEY` as a secret.",
        )
        return

    # ── Check agent imported correctly ──
    if not AGENT_OK:
        yield (
            "", "", "", "", "", None, None, None, "",
            f"❌ Failed to import agent:\n```\n{AGENT_ERR}\n```",
        )
        return

    # ── Run agent in background thread, stream logs ──
    log_lines = []
    log_queue: Queue = Queue()
    pipeline_done = threading.Event()
    pipeline_result: dict = {}

    def log_fn(msg: str):
        log_queue.put(msg)

    def run_thread():
        try:
            pipeline_result["data"] = run_topic_modelling_agent(csv_text, log=log_fn)
        except Exception as exc:
            pipeline_result["error"] = (
                f"{type(exc).__name__}: {exc}\n\n{traceback.format_exc()}"
            )
        finally:
            pipeline_done.set()

    threading.Thread(target=run_thread, daemon=True).start()

    # Stream logs until done
    while not pipeline_done.is_set() or not log_queue.empty():
        drained = False
        while not log_queue.empty():
            log_lines.append(log_queue.get_nowait())
            drained = True
        if drained:
            yield ("\n".join(log_lines), "", "", "", "", None, None, None, "", "")
        else:
            time.sleep(0.4)

    # Drain any remaining log messages
    while not log_queue.empty():
        log_lines.append(log_queue.get_nowait())

    # ── Handle error ──
    if "error" in pipeline_result:
        err = pipeline_result["error"]
        yield (
            "\n".join(log_lines), "", "", "", "", None, None, None, "",
            f"❌ Agent error:\n```\n{err}\n```",
        )
        return

    # ── Build all outputs ──
    data = pipeline_result["data"]

    topics_md = make_topics_markdown(data["frequency_table"])
    comparison_md = make_comparison_markdown(data["comparison"])
    pajais_md = make_pajais_markdown(data["taxonomy_result"])
    narrative = data["narrative"]
    summary_md = make_summary_markdown(data["summary"])

    # Write download files to /tmp
    import tempfile
    tmp = tempfile.mkdtemp()

    comparison_csv_path = os.path.join(tmp, "comparison.csv")
    with open(comparison_csv_path, "w", encoding="utf-8") as f:
        f.write(data["comparison_csv"])

    taxonomy_json_path = os.path.join(tmp, "taxonomy_map.json")
    with open(taxonomy_json_path, "w", encoding="utf-8") as f:
        f.write(data["taxonomy_json"])

    narrative_txt_path = os.path.join(tmp, "narrative.txt")
    with open(narrative_txt_path, "w", encoding="utf-8") as f:
        f.write(narrative)

    final_log = "\n".join(log_lines) + "\n\n✅ All outputs ready — check the tabs above!"

    yield (
        final_log,
        topics_md,
        comparison_md,
        pajais_md,
        narrative,
        comparison_csv_path,
        taxonomy_json_path,
        narrative_txt_path,
        summary_md,
        "",   # no error
    )


# ── CSS ───────────────────────────────────────────────────────────────────────

CSS = """
@import url('https://fonts.googleapis.com/css2?family=Playfair+Display:wght@700;900&family=IBM+Plex+Mono:wght@400;500&family=Inter:wght@300;400;500;600&display=swap');

:root {
    --bg:        #0d1117;
    --surface:   #161b22;
    --card:      #1c2230;
    --border:    #30363d;
    --accent:    #58a6ff;
    --accent2:   #3fb950;
    --accent3:   #d2a8ff;
    --text:      #c9d1d9;
    --muted:     #8b949e;
    --danger:    #f85149;
}

body, .gradio-container {
    background: var(--bg) !important;
    color: var(--text) !important;
    font-family: 'Inter', sans-serif !important;
}

.app-header {
    padding: 2rem 1.5rem 1.2rem;
    text-align: center;
    background: linear-gradient(180deg, #161b22 0%, #0d1117 100%);
    border-bottom: 1px solid var(--border);
    margin-bottom: 1rem;
}
.app-title {
    font-family: 'Playfair Display', serif;
    font-size: 2.2rem;
    font-weight: 900;
    color: var(--accent);
    margin: 0 0 0.3rem;
    letter-spacing: -0.02em;
}
.app-subtitle {
    font-family: 'IBM Plex Mono', monospace;
    font-size: 0.82rem;
    color: var(--muted);
    margin: 0;
}

.section-label {
    font-size: 0.75rem;
    font-weight: 600;
    letter-spacing: 0.08em;
    text-transform: uppercase;
    color: var(--muted);
    margin: 0.8rem 0 0.3rem;
}

button.primary {
    background: linear-gradient(135deg, #1f6feb 0%, #388bfd 100%) !important;
    color: #fff !important;
    font-weight: 600 !important;
    border: none !important;
    border-radius: 8px !important;
    font-size: 1rem !important;
}
button.secondary {
    background: var(--card) !important;
    color: var(--text) !important;
    border: 1px solid var(--border) !important;
    border-radius: 8px !important;
}

textarea, input[type=text] {
    background: var(--card) !important;
    border: 1px solid var(--border) !important;
    color: var(--text) !important;
    font-family: 'IBM Plex Mono', monospace !important;
    font-size: 0.82rem !important;
    border-radius: 8px !important;
}

.tab-nav button {
    font-family: 'Inter', sans-serif !important;
    font-size: 0.85rem !important;
}

footer { display: none !important; }
"""

# ── Clear helper (module-level so it's reachable by tests and Gradio) ────────

def clear_all():
    """Reset all UI outputs to their default empty state."""
    return (
        "",                                         # log_box
        "_Run the agent to see topics._",           # topics_md
        "_Run the agent to see comparison._",       # comparison_md
        "_Run the agent to see PAJAIS mapping._",   # pajais_md
        "",                                         # narrative_txt
        None,                                       # dl_csv
        None,                                       # dl_json
        None,                                       # dl_txt
        "_Summary will appear here after the run._",  # summary_md
        "",                                         # err_md
    )


# ── Build UI ──────────────────────────────────────────────────────────────────

def build_ui() -> gr.Blocks:
    with gr.Blocks(title="Journal Topic Modelling Agent", css=CSS) as demo:

        gr.HTML("""
        <div class="app-header">
          <div class="app-title">📚 Journal Topic Modelling Agent</div>
          <div class="app-subtitle">
            Mistral AI &nbsp;·&nbsp; 5-Phase Agentic Pipeline &nbsp;·&nbsp;
            PAJAIS Taxonomy &nbsp;·&nbsp; Title vs Abstract Analysis
          </div>
        </div>
        """)

        with gr.Row(equal_height=False):

            # ── Left panel ────────────────────────────────────────────────
            with gr.Column(scale=1, min_width=300):

                gr.HTML('<div class="section-label">📂 Input</div>')

                csv_file = gr.File(
                    label="Upload journal CSV",
                    file_types=[".csv"],
                    type="filepath",
                )
                csv_text = gr.Textbox(
                    label="Or paste CSV text",
                    lines=5,
                    placeholder='title,abstract,year\n"My Paper","Abstract text…",2023',
                )
                use_sample = gr.Checkbox(
                    label="🧪 Use built-in sample (15 papers)",
                    value=False,
                )

                with gr.Row():
                    run_btn = gr.Button("🚀 Run Agent", variant="primary", size="lg")
                    clear_btn = gr.Button("🗑️ Clear", variant="secondary", size="sm")

                gr.HTML('<div class="section-label">📥 Downloads</div>')
                dl_csv  = gr.File(label="comparison.csv",     interactive=False)
                dl_json = gr.File(label="taxonomy_map.json",  interactive=False)
                dl_txt  = gr.File(label="narrative.txt",      interactive=False)

                gr.HTML('<div class="section-label">📊 Run Summary</div>')
                summary_md = gr.Markdown("_Summary will appear here after the run._")

            # ── Right panel ───────────────────────────────────────────────
            with gr.Column(scale=2):

                err_md = gr.Markdown(value="", label="")

                with gr.Tabs():

                    with gr.Tab("📋 Agent Log"):
                        log_box = gr.Textbox(
                            label="Live log",
                            lines=22,
                            max_lines=30,
                            interactive=False,
                            placeholder="Agent progress will stream here…",
                            show_copy_button=True,
                        )

                    with gr.Tab("🏷️ Topics — Phase 2"):
                        gr.Markdown(
                            "All discovered topics with labels and document frequency counts. "
                            "Sorted by frequency (highest first)."
                        )
                        topics_md = gr.Markdown("_Run the agent to see topics._")

                    with gr.Tab("⚖️ Comparison — Phase 3"):
                        gr.Markdown(
                            "Title vs Abstract topic distribution. "
                            "**Δ** = how much more prominent a topic is in abstracts vs titles. "
                            "▲ = more abstract-prominent, ▼ = more title-prominent."
                        )
                        comparison_md = gr.Markdown("_Run the agent to see comparison._")

                    with gr.Tab("🗺️ PAJAIS Map — Phase 4"):
                        gr.Markdown(
                            "Topics mapped to the PAJAIS taxonomy. "
                            "**NOVEL** = no matching PAJAIS theme — potential publication gaps."
                        )
                        pajais_md = gr.Markdown("_Run the agent to see PAJAIS mapping._")

                    with gr.Tab("✍️ Narrative — Phase 5"):
                        gr.Markdown(
                            "Auto-generated 500+ word **Section 7 Discussion & Conclusions** draft. "
                            "Editable — you can refine it directly here."
                        )
                        narrative_txt = gr.Textbox(
                            label="Section 7 draft (editable)",
                            lines=22,
                            interactive=True,
                            placeholder="Narrative will appear here after the run…",
                            show_copy_button=True,
                        )

        # ── Output list (must match run_pipeline yield order exactly) ─────
        ALL_OUTPUTS = [
            log_box,        # 0
            topics_md,      # 1
            comparison_md,  # 2
            pajais_md,      # 3
            narrative_txt,  # 4
            dl_csv,         # 5
            dl_json,        # 6
            dl_txt,         # 7
            summary_md,     # 8
            err_md,         # 9
        ]

        # ── Run button ────────────────────────────────────────────────────
        run_btn.click(
            fn=run_pipeline,
            inputs=[csv_file, csv_text, use_sample],
            outputs=ALL_OUTPUTS,
        )

        # ── Clear button ──────────────────────────────────────────────────
        clear_btn.click(fn=clear_all, outputs=ALL_OUTPUTS)

        gr.HTML(
            '<div style="text-align:center;color:#8b949e;font-size:0.78rem;'
            'padding:1.2rem 0 0.5rem;font-family:\'IBM Plex Mono\',monospace;">'
            "Outputs: labelled topics · frequency table · comparison.csv · "
            "taxonomy_map.json · narrative.txt (500+ words)</div>"
        )

    return demo


# ── Entry point ───────────────────────────────────────────────────────────────

if __name__ == "__main__":
    demo = build_ui()
    demo.queue()   # Required for generator streaming
    demo.launch(server_name="0.0.0.0", server_port=7860, share=False)