import datetime as dt import html import json import os import threading from pathlib import Path import gradio as gr from huggingface_hub import hf_hub_download, upload_file from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError from scoring import ( SubmissionError, leaderboard_identity, leaderboard_row, load_jsonl_text, parse_submission_text, rank_leaderboard, safe_slug, score_predictions, ) PUBLIC_DATASET_REPO = os.getenv("PUBLIC_DATASET_REPO", "amitbcp/docinsights-2026-shared-task-data") WORKSHOP_URL = os.getenv( "WORKSHOP_URL", "https://docinsights-workshop.github.io/docinsights-2026/shared-task/", ) SOURCE_REPO_URL = os.getenv("SOURCE_REPO_URL", "https://github.com/oracle-samples/gsm-sem") PARTICIPANT_GUIDE_URL = os.getenv( "PARTICIPANT_GUIDE_URL", "https://github.com/oracle-samples/gsm-sem/blob/main/docsem/PARTICIPANT_INSTRUCTIONS.md", ) GOLD_REPO_ID = os.getenv("GOLD_REPO_ID", "amitbcp/docinsights-2026-shared-task-submissions") GOLD_FILE = os.getenv("GOLD_FILE", "private/val_labels.jsonl") SUBMISSIONS_REPO_ID = os.getenv("SUBMISSIONS_REPO_ID", GOLD_REPO_ID) WRITE_TOKEN = os.getenv("HF_WRITE_TOKEN") or os.getenv("HF_TOKEN") GRADIO_MAJOR_VERSION = int(gr.__version__.split(".", maxsplit=1)[0]) LEADERBOARD_LOCK = threading.Lock() PORTAL_CSS = """ :root { --docsem-navy: #17365f; --docsem-teal: #177f78; --docsem-coral: #cc4b2c; --docsem-gold: #d9a62e; --docsem-ink: #17212f; --docsem-muted: #5d6878; --docsem-line: #d9dee5; --docsem-surface: #ffffff; --docsem-page: #f7f8fa; } .gradio-container { width: 100% !important; max-width: 1540px !important; margin: 0 auto !important; padding: 24px 30px 36px !important; background: var(--docsem-page); color: var(--docsem-ink); font-size: 16px; } #portal-header { display: flex; align-items: center; justify-content: space-between; gap: 24px; padding: 6px 0 24px; border-bottom: 1px solid var(--docsem-line); margin-bottom: 20px; } #portal-header .portal-kicker { display: block; color: var(--docsem-coral); font-size: 13px; font-weight: 750; line-height: 1.3; letter-spacing: 0; margin-bottom: 6px; text-transform: uppercase; } #portal-header h1 { color: var(--docsem-navy); font-size: 36px; line-height: 1.15; letter-spacing: 0; margin: 0 0 8px; } #portal-header p { color: var(--docsem-muted); font-size: 16px; line-height: 1.5; margin: 0; } #portal-header .portal-summary { color: var(--docsem-ink); font-size: 18px; font-weight: 650; margin-bottom: 4px; } #portal-header .portal-links { display: flex; align-items: center; flex-wrap: nowrap; justify-content: flex-end; gap: 10px; } #portal-header a { display: inline-flex; align-items: center; justify-content: center; min-height: 44px; padding: 0 16px; color: var(--docsem-navy); background: var(--docsem-surface); border: 1px solid #b9c4d1; border-radius: 6px; font-size: 15px; font-weight: 700; text-decoration: none; white-space: nowrap; } #portal-header a.primary-link { color: #ffffff; background: var(--docsem-navy); border-color: var(--docsem-navy); } #portal-header a:hover { border-color: var(--docsem-teal); color: var(--docsem-teal); } #portal-header a.primary-link:hover { color: #ffffff; background: var(--docsem-teal); } .column > #submission-panel { padding: 18px 20px 16px; background: var(--docsem-surface); border: 1px solid var(--docsem-line); border-left: 4px solid var(--docsem-coral); border-radius: 6px; } #submission-panel #submission-panel { padding: 0; background: transparent; border: 0; border-radius: 0; } #submission-panel .styler { background: transparent; } #submission-panel h2 { color: var(--docsem-ink); font-size: 23px; line-height: 1.25; letter-spacing: 0; margin: 0 0 5px; } #submission-panel .submission-note { color: var(--docsem-muted); font-size: 15px; line-height: 1.5; margin: 0 0 12px; } #submission-panel .submission-note a { color: var(--docsem-teal); font-weight: 700; text-decoration: none; } #submission-panel .submission-note a:hover { text-decoration: underline; } #submission-fields { gap: 14px; } #submission-fields .form { border: 0; } #submission-fields label span { color: #465263; font-size: 14px; font-weight: 600; } #submission-fields input { font-size: 16px; } #submission-actions { align-items: stretch; gap: 16px; margin-top: 8px; } #submission-file { min-height: 112px !important; height: 112px !important; } #submission-file > div { min-height: 110px !important; } #submission-file { font-size: 15px; } #submission-side { justify-content: center; padding: 2px 0; } #submission-side p { color: var(--docsem-muted); font-size: 14px; line-height: 1.5; margin: 0 0 10px; } #submit-button { min-height: 46px; border-radius: 6px; font-size: 16px; font-weight: 700; } #score-output { max-height: 178px; margin-top: 14px; overflow: auto; } #leaderboard-section { margin-top: 28px; } #leaderboard-heading { align-items: center; justify-content: space-between; gap: 18px; margin-bottom: 10px; } #leaderboard-heading h2 { color: var(--docsem-navy); font-size: 29px; line-height: 1.2; letter-spacing: 0; margin: 0 0 6px; } #leaderboard-heading p { color: var(--docsem-muted); font-size: 15px; line-height: 1.45; margin: 0; } #refresh-button { max-width: 170px; min-height: 42px; border-radius: 6px; font-size: 15px; font-weight: 700; } #leaderboard-table { background: var(--docsem-surface); border: 1px solid var(--docsem-line); border-radius: 6px; overflow: hidden; } #leaderboard-table .leaderboard-table-wrap { width: 100%; overflow-x: auto; } #leaderboard-table table { width: 100%; min-width: 920px; border-collapse: collapse; table-layout: fixed; font-size: 15px; } #leaderboard-table th { color: var(--docsem-navy); background: #eef2f6; font-size: 14px; font-weight: 700; line-height: 1.25; white-space: normal; text-align: left; } #leaderboard-table td, #leaderboard-table th { padding: 13px 14px; border-bottom: 1px solid var(--docsem-line); vertical-align: middle; } #leaderboard-table td { color: var(--docsem-ink); line-height: 1.35; overflow-wrap: anywhere; } #leaderboard-table tbody tr:last-child td { border-bottom: 0; } #leaderboard-table tbody tr:hover { background: #f8fafc; } #leaderboard-table .leaderboard-rank, #leaderboard-table .leaderboard-attempts, #leaderboard-table .leaderboard-metric { text-align: center; } #leaderboard-table .leaderboard-metric { color: var(--docsem-navy); font-variant-numeric: tabular-nums; font-weight: 700; white-space: nowrap; } #leaderboard-table .leaderboard-date { color: var(--docsem-muted); font-variant-numeric: tabular-nums; white-space: nowrap; } #leaderboard-table .leaderboard-empty { padding: 34px 20px; color: var(--docsem-muted); text-align: center; } @media (max-width: 760px) { .gradio-container { padding: 16px 14px 26px !important; } #portal-header { align-items: flex-start; flex-direction: column; gap: 14px; } #portal-header h1 { font-size: 28px; } #portal-header .portal-summary { font-size: 17px; } #portal-header .portal-links { width: 100%; } #portal-header a { flex: 1 1 140px; min-width: 0; padding: 0 10px; } .column > #submission-panel { padding: 16px 14px; } #leaderboard-heading { align-items: flex-start; flex-direction: column; } #refresh-button { max-width: none; } } """ def _read_hub_file(repo_id, filename, token=None, force_download=False): path = hf_hub_download( repo_id=repo_id, filename=filename, repo_type="dataset", token=token, force_download=force_download, ) return Path(path).read_text(encoding="utf-8") def _load_gold_rows(): return load_jsonl_text(_read_hub_file(GOLD_REPO_ID, GOLD_FILE, token=WRITE_TOKEN)) def _persist_submission(rows, team, contact, submission_name, metrics): if not SUBMISSIONS_REPO_ID or not WRITE_TOKEN: return "Score computed. Persistence is disabled until SUBMISSIONS_REPO_ID and HF_WRITE_TOKEN are configured." submitted_at = dt.datetime.utcnow().replace(microsecond=0).isoformat() + "Z" row = leaderboard_row(team, contact, submission_name, metrics, submitted_at) payload = { "leaderboard": row, "metrics": metrics, "predictions": rows, } timestamp = submitted_at.replace(":", "").replace("-", "") team_slug = safe_slug(team) name_slug = safe_slug(submission_name) filename = f"{timestamp}_{team_slug}_{name_slug}.json" tmp_path = Path("/tmp") / filename tmp_path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") upload_file( path_or_fileobj=str(tmp_path), path_in_repo=f"submissions/{filename}", repo_id=SUBMISSIONS_REPO_ID, repo_type="dataset", token=WRITE_TOKEN, commit_message=f"Add submission {team_slug}/{name_slug}", ) attempts = _update_leaderboard(row) return ( f"Score computed and saved to {SUBMISSIONS_REPO_ID}/submissions/{filename}. " f"This is attempt {attempts} for this team and contact email." ) def _load_leaderboard_rows(): if not SUBMISSIONS_REPO_ID or not WRITE_TOKEN: return [] try: text = _read_hub_file( SUBMISSIONS_REPO_ID, "leaderboard/leaderboard.json", token=WRITE_TOKEN, force_download=True, ) except (EntryNotFoundError, RepositoryNotFoundError): return [] rows = json.loads(text) return rows if isinstance(rows, list) else [] def _sort_leaderboard(rows): return rank_leaderboard(rows) def _update_leaderboard(row): with LEADERBOARD_LOCK: rows = _load_leaderboard_rows() rows.append(row) rows.sort(key=lambda item: str(item.get("submitted_at", ""))) tmp_path = Path("/tmp") / "leaderboard.json" tmp_path.write_text(json.dumps(rows, indent=2, sort_keys=True), encoding="utf-8") upload_file( path_or_fileobj=str(tmp_path), path_in_repo="leaderboard/leaderboard.json", repo_id=SUBMISSIONS_REPO_ID, repo_type="dataset", token=WRITE_TOKEN, commit_message="Update leaderboard", ) identity = leaderboard_identity(row) best = next( item for item in rank_leaderboard(rows) if leaderboard_identity(item) == identity ) return best["attempts"] def _format_metric(value): return f"{float(value) * 100:.2f}%" def _format_timestamp(value): return str(value).replace("T", " ").removesuffix("Z") def leaderboard_html(): rows = _sort_leaderboard(_load_leaderboard_rows()) body_rows = [] for index, row in enumerate(rows[:100], start=1): body_rows.append( f""" {index} {html.escape(str(row.get("team", "")))} {html.escape(str(row.get("submission_name", "")))} {int(row.get("attempts", 1))} {_format_metric(row.get("answer_accuracy", 0.0))} {_format_metric(row.get("evidence_f1", 0.0))} {html.escape(_format_timestamp(row.get("submitted_at", "")))} """ ) if not body_rows: body_rows.append( 'No scored submissions yet.' ) return f"""
{''.join(body_rows)}
Rank Team Best submission Attempts Answer accuracy Evidence F1 Best score (UTC)
""" def evaluate_submission(file_obj, team, contact, submission_name): if file_obj is None: raise gr.Error("Upload a JSONL submission file.") if not team.strip(): raise gr.Error("Enter a team name.") if not contact.strip(): raise gr.Error("Enter a contact email.") if not submission_name.strip(): raise gr.Error("Enter a submission name.") try: text = Path(file_obj.name).read_text(encoding="utf-8") rows = parse_submission_text(text) labels = _load_gold_rows() metrics = score_predictions(rows, labels) message = _persist_submission(rows, team.strip(), contact.strip(), submission_name.strip(), metrics) except SubmissionError as exc: raise gr.Error(str(exc)) from exc except Exception as exc: raise gr.Error(f"Could not score submission: {exc}") from exc return gr.update( value={ "answer_accuracy": metrics["answer_accuracy"], "evidence_exact_match": metrics["evidence_exact_match"], "evidence_f1": metrics["evidence_f1"], "examples": metrics["examples"], "message": message, }, visible=True, ) blocks_options = { "title": "DocInsights 2026 Shared Task: DocSem", "fill_width": True, } if GRADIO_MAJOR_VERSION < 6: blocks_options["css"] = PORTAL_CSS with gr.Blocks(**blocks_options) as demo: gr.HTML( f"""
Workshop on Document Intelligence and Understanding

DocInsights 2026 Shared Task: DocSem

Document-grounded quantitative reasoning with evidence attribution.

Co-located with EMNLP 2026 in Budapest, Hungary. Beyond Plain Text: Bridging NLP and Document AI.

""" ) with gr.Group(elem_id="submission-panel"): gr.HTML( f"""

Submit validation predictions

Upload one JSON object per instance with instance_id, answer, and evidence. Review the participant guide for the complete format and evaluation protocol.

""" ) with gr.Row(elem_id="submission-fields"): team = gr.Textbox(label="Team", placeholder="example-team") contact = gr.Textbox(label="Contact email", placeholder="lead@example.org") submission_name = gr.Textbox(label="Submission name", placeholder="baseline-v1") with gr.Row(elem_id="submission-actions"): file_input = gr.File( label="Submission file", file_types=[".jsonl", ".json"], height=112, scale=4, elem_id="submission-file", ) with gr.Column(scale=1, min_width=210, elem_id="submission-side"): gr.Markdown( "Accepted: `.jsonl` or `.json` \n" "Validation labels remain organizer-only." ) submit = gr.Button( "Validate and score", variant="primary", elem_id="submit-button", ) result = gr.JSON( label="Submission score", visible=False, height=160, elem_id="score-output", ) submit.click( evaluate_submission, inputs=[file_input, team, contact, submission_name], outputs=result, ) with gr.Column(elem_id="leaderboard-section"): with gr.Row(elem_id="leaderboard-heading"): gr.HTML( """

Leaderboard

Best attempt per team and contact email. Ranked by answer accuracy, then evidence F1. Attempt counts include every valid submission.

""" ) refresh = gr.Button( "Refresh results", variant="secondary", scale=0, min_width=170, elem_id="refresh-button", ) leaderboard = gr.HTML( value=leaderboard_html, elem_id="leaderboard-table", ) refresh.click(leaderboard_html, inputs=None, outputs=leaderboard) if __name__ == "__main__": launch_options = {"css": PORTAL_CSS} if GRADIO_MAJOR_VERSION >= 6 else {} demo.launch(**launch_options)