app.py

"""
ILSA-Survey-Dataset — Clickable Source Viewer
"""

import gradio as gr
import pandas as pd
from huggingface_hub import hf_hub_download

REPO_ID = "dedemerve/ILSA-Survey-Dataset"
MAX_CELL_CHARS = 300

_CACHE = {}

SHEETS = {
    "Articles (130 studies)": "data/articles_master.csv",
    "Main Findings (202 outcomes)": "data/main_findings.csv",
    "Confounders (1907 predictors)": "data/confounders.csv",
}


def _is_blank(val) -> bool:
    if val is None:
        return True
    try:
        if pd.isna(val):
            return True
    except (TypeError, ValueError):
        pass
    return str(val).strip().lower() in ("", "none", "null", "nan", "n/a", "<na>")


def _truncate(val) -> str:
    if _is_blank(val):
        return ""
    s = str(val)
    return s if len(s) <= MAX_CELL_CHARS else s[:MAX_CELL_CHARS] + "…"


_LONGTEXT_HINTS = ("interpretation", "summary", "description", "definition", "finding",
                   "confounder", "notes", "abstract", "text", "criteria", "explanation",
                   "outcome", "primary", "technique", "filter")
_SHORT_HINTS = ("year", "doi", "type", "category", "used", "id", "url", "label", "size")


def _column_width(col: str) -> str:
    c = col.lower()
    if col == "Source":
        return "200px"
    if "title" in c or c in ("name",):
        return "280px"
    if "journal" in c or "venue" in c or "authors" in c:
        return "220px"
    if any(h in c for h in _LONGTEXT_HINTS):
        return "300px"
    if any(h in c for h in _SHORT_HINTS):
        return "110px"
    return "160px"


def _load_raw(sheet_key: str) -> pd.DataFrame:
    if sheet_key not in _CACHE:
        csv_path = SHEETS[sheet_key]
        local = hf_hub_download(repo_id=REPO_ID, filename=csv_path, repo_type="dataset")
        _CACHE[sheet_key] = pd.read_csv(local, dtype=str)
    return _CACHE[sheet_key]


def _make_source_link(row) -> str:
    for col in ("source_url", "paper_url"):
        val = row.get(col)
        if not _is_blank(val):
            url = str(val).strip()
            return f"[Open paper]({url})"
    doi = row.get("doi")
    if not _is_blank(doi):
        doi = str(doi).strip()
        url = doi if doi.startswith("http") else f"https://doi.org/{doi}"
        return f"[Open via DOI]({url})"
    import urllib.parse
    title = row.get("title", "")
    if not _is_blank(title):
        q = urllib.parse.quote_plus(str(title).strip())
        return f"[Search Google Scholar](https://scholar.google.com/scholar?q={q})"
    return ""


def build_table(sheet_key: str, search_text: str, max_rows: int):
    if not sheet_key:
        return gr.update(), "Select a table to get started."
    try:
        df = _load_raw(sheet_key).copy()
    except Exception as e:
        return gr.update(), f"Could not load data: {e}"

    total_rows = len(df)

    if search_text:
        title_col = next((c for c in df.columns if "title" in c.lower()), None)
        if title_col:
            df = df[df[title_col].astype(str).str.contains(search_text, case=False, na=False)]

    filtered_rows = len(df)
    df = df.head(max_rows)

    # Build Source link column if link-related columns exist
    link_cols = [c for c in df.columns if c in ("source_url", "paper_url", "doi")]
    has_links = bool(link_cols)

    if has_links:
        source_col = df.apply(_make_source_link, axis=1)
        cols_to_drop = [c for c in ("source_url", "paper_url") if c in df.columns]
        df = df.drop(columns=cols_to_drop)
        df.insert(0, "Source", source_col)

    for col in df.columns:
        if col == "Source":
            continue
        df[col] = df[col].apply(_truncate)

    datatype = ["markdown" if col == "Source" else "str" for col in df.columns]
    column_widths = [_column_width(col) for col in df.columns]

    info = (
        f"**{sheet_key}** — {total_rows} rows total, "
        f"{filtered_rows} after filtering, showing {len(df)} rows."
    )
    if has_links:
        info += " Click **Source** to open the paper."

    return gr.update(value=df, datatype=datatype, column_widths=column_widths), info


with gr.Blocks(title="ILSA Survey Dataset Viewer") as demo:
    gr.Markdown(
        f"""
        # ILSA Survey Dataset — Clickable Source Viewer

        Browse the three relational tables from the survey paper
        *"Artificial Intelligence Applications in International Large-Scale Assessments:
        A Survey with LLM-Assisted Evidence Synthesis"* (Dede & Çetinkaya, 2026).

        Each row in the **Articles** table links directly to the paper via DOI.

        Dataset: [`{REPO_ID}`](https://huggingface.co/datasets/{REPO_ID}) &nbsp;|&nbsp;
        Website: [dedemerve.github.io/ILSA-Survey-Extractor](https://dedemerve.github.io/ILSA-Survey-Extractor/)
        """
    )
    with gr.Row():
        sheet_dd = gr.Dropdown(
            choices=list(SHEETS.keys()),
            value=list(SHEETS.keys())[0],
            label="Table",
        )
    with gr.Row():
        search_box = gr.Textbox(
            label="Search by title (Articles table only)",
            placeholder="e.g. PISA, reading, ICCS…",
        )
        max_rows_box = gr.Slider(minimum=20, maximum=2000, value=200, step=20, label="Max rows")
        load_btn = gr.Button("Load / Filter", variant="primary")

    status_md = gr.Markdown("Loading…")
    table = gr.Dataframe(label="Results", wrap=True, datatype="str", max_height=650)

    demo.load(build_table, inputs=[sheet_dd, search_box, max_rows_box], outputs=[table, status_md])
    sheet_dd.change(build_table, inputs=[sheet_dd, search_box, max_rows_box], outputs=[table, status_md])
    load_btn.click(build_table, inputs=[sheet_dd, search_box, max_rows_box], outputs=[table, status_md])
    search_box.submit(build_table, inputs=[sheet_dd, search_box, max_rows_box], outputs=[table, status_md])

if __name__ == "__main__":
    demo.launch()