import datetime as dt import html import json import os import threading from pathlib import Path import gradio as gr from huggingface_hub import hf_hub_download, upload_file from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError from scoring import ( SubmissionError, leaderboard_identity, leaderboard_row, load_jsonl_text, parse_submission_text, rank_leaderboard, safe_slug, score_predictions, ) PUBLIC_DATASET_REPO = os.getenv("PUBLIC_DATASET_REPO", "amitbcp/docinsights-2026-shared-task-data") WORKSHOP_URL = os.getenv( "WORKSHOP_URL", "https://docinsights-workshop.github.io/docinsights-2026/shared-task/", ) SOURCE_REPO_URL = os.getenv("SOURCE_REPO_URL", "https://github.com/oracle-samples/gsm-sem") PARTICIPANT_GUIDE_URL = os.getenv( "PARTICIPANT_GUIDE_URL", "https://github.com/oracle-samples/gsm-sem/blob/main/docsem/PARTICIPANT_INSTRUCTIONS.md", ) GOLD_REPO_ID = os.getenv("GOLD_REPO_ID", "amitbcp/docinsights-2026-shared-task-submissions") GOLD_FILE = os.getenv("GOLD_FILE", "private/val_labels.jsonl") SUBMISSIONS_REPO_ID = os.getenv("SUBMISSIONS_REPO_ID", GOLD_REPO_ID) WRITE_TOKEN = os.getenv("HF_WRITE_TOKEN") or os.getenv("HF_TOKEN") GRADIO_MAJOR_VERSION = int(gr.__version__.split(".", maxsplit=1)[0]) LEADERBOARD_LOCK = threading.Lock() PORTAL_CSS = """ :root { --docsem-navy: #17365f; --docsem-teal: #177f78; --docsem-coral: #cc4b2c; --docsem-gold: #d9a62e; --docsem-ink: #17212f; --docsem-muted: #5d6878; --docsem-line: #d9dee5; --docsem-surface: #ffffff; --docsem-page: #f7f8fa; } .gradio-container { width: 100% !important; max-width: 1540px !important; margin: 0 auto !important; padding: 24px 30px 36px !important; background: var(--docsem-page); color: var(--docsem-ink); font-size: 16px; } #portal-header { display: flex; align-items: center; justify-content: space-between; gap: 24px; padding: 6px 0 24px; border-bottom: 1px solid var(--docsem-line); margin-bottom: 20px; } #portal-header .portal-kicker { display: block; color: var(--docsem-coral); font-size: 13px; font-weight: 750; line-height: 1.3; letter-spacing: 0; margin-bottom: 6px; text-transform: uppercase; } #portal-header h1 { color: var(--docsem-navy); font-size: 36px; line-height: 1.15; letter-spacing: 0; margin: 0 0 8px; } #portal-header p { color: var(--docsem-muted); font-size: 16px; line-height: 1.5; margin: 0; } #portal-header .portal-summary { color: var(--docsem-ink); font-size: 18px; font-weight: 650; margin-bottom: 4px; } #portal-header .portal-links { display: flex; align-items: center; flex-wrap: nowrap; justify-content: flex-end; gap: 10px; } #portal-header a { display: inline-flex; align-items: center; justify-content: center; min-height: 44px; padding: 0 16px; color: var(--docsem-navy); background: var(--docsem-surface); border: 1px solid #b9c4d1; border-radius: 6px; font-size: 15px; font-weight: 700; text-decoration: none; white-space: nowrap; } #portal-header a.primary-link { color: #ffffff; background: var(--docsem-navy); border-color: var(--docsem-navy); } #portal-header a:hover { border-color: var(--docsem-teal); color: var(--docsem-teal); } #portal-header a.primary-link:hover { color: #ffffff; background: var(--docsem-teal); } .column > #submission-panel { padding: 18px 20px 16px; background: var(--docsem-surface); border: 1px solid var(--docsem-line); border-left: 4px solid var(--docsem-coral); border-radius: 6px; } #submission-panel #submission-panel { padding: 0; background: transparent; border: 0; border-radius: 0; } #submission-panel .styler { background: transparent; } #submission-panel h2 { color: var(--docsem-ink); font-size: 23px; line-height: 1.25; letter-spacing: 0; margin: 0 0 5px; } #submission-panel .submission-note { color: var(--docsem-muted); font-size: 15px; line-height: 1.5; margin: 0 0 12px; } #submission-panel .submission-note a { color: var(--docsem-teal); font-weight: 700; text-decoration: none; } #submission-panel .submission-note a:hover { text-decoration: underline; } #submission-fields { gap: 14px; } #submission-fields .form { border: 0; } #submission-fields label span { color: #465263; font-size: 14px; font-weight: 600; } #submission-fields input { font-size: 16px; } #submission-actions { align-items: stretch; gap: 16px; margin-top: 8px; } #submission-file { min-height: 112px !important; height: 112px !important; } #submission-file > div { min-height: 110px !important; } #submission-file { font-size: 15px; } #submission-side { justify-content: center; padding: 2px 0; } #submission-side p { color: var(--docsem-muted); font-size: 14px; line-height: 1.5; margin: 0 0 10px; } #submit-button { min-height: 46px; border-radius: 6px; font-size: 16px; font-weight: 700; } #score-output { max-height: 178px; margin-top: 14px; overflow: auto; } #leaderboard-section { margin-top: 28px; } #leaderboard-heading { align-items: center; justify-content: space-between; gap: 18px; margin-bottom: 10px; } #leaderboard-heading h2 { color: var(--docsem-navy); font-size: 29px; line-height: 1.2; letter-spacing: 0; margin: 0 0 6px; } #leaderboard-heading p { color: var(--docsem-muted); font-size: 15px; line-height: 1.45; margin: 0; } #refresh-button { max-width: 170px; min-height: 42px; border-radius: 6px; font-size: 15px; font-weight: 700; } #leaderboard-table { background: var(--docsem-surface); border: 1px solid var(--docsem-line); border-radius: 6px; overflow: hidden; } #leaderboard-table .leaderboard-table-wrap { width: 100%; overflow-x: auto; } #leaderboard-table table { width: 100%; min-width: 920px; border-collapse: collapse; table-layout: fixed; font-size: 15px; } #leaderboard-table th { color: var(--docsem-navy); background: #eef2f6; font-size: 14px; font-weight: 700; line-height: 1.25; white-space: normal; text-align: left; } #leaderboard-table td, #leaderboard-table th { padding: 13px 14px; border-bottom: 1px solid var(--docsem-line); vertical-align: middle; } #leaderboard-table td { color: var(--docsem-ink); line-height: 1.35; overflow-wrap: anywhere; } #leaderboard-table tbody tr:last-child td { border-bottom: 0; } #leaderboard-table tbody tr:hover { background: #f8fafc; } #leaderboard-table .leaderboard-rank, #leaderboard-table .leaderboard-attempts, #leaderboard-table .leaderboard-metric { text-align: center; } #leaderboard-table .leaderboard-metric { color: var(--docsem-navy); font-variant-numeric: tabular-nums; font-weight: 700; white-space: nowrap; } #leaderboard-table .leaderboard-date { color: var(--docsem-muted); font-variant-numeric: tabular-nums; white-space: nowrap; } #leaderboard-table .leaderboard-empty { padding: 34px 20px; color: var(--docsem-muted); text-align: center; } @media (max-width: 760px) { .gradio-container { padding: 16px 14px 26px !important; } #portal-header { align-items: flex-start; flex-direction: column; gap: 14px; } #portal-header h1 { font-size: 28px; } #portal-header .portal-summary { font-size: 17px; } #portal-header .portal-links { width: 100%; } #portal-header a { flex: 1 1 140px; min-width: 0; padding: 0 10px; } .column > #submission-panel { padding: 16px 14px; } #leaderboard-heading { align-items: flex-start; flex-direction: column; } #refresh-button { max-width: none; } } """ def _read_hub_file(repo_id, filename, token=None, force_download=False): path = hf_hub_download( repo_id=repo_id, filename=filename, repo_type="dataset", token=token, force_download=force_download, ) return Path(path).read_text(encoding="utf-8") def _load_gold_rows(): return load_jsonl_text(_read_hub_file(GOLD_REPO_ID, GOLD_FILE, token=WRITE_TOKEN)) def _persist_submission(rows, team, contact, submission_name, metrics): if not SUBMISSIONS_REPO_ID or not WRITE_TOKEN: return "Score computed. Persistence is disabled until SUBMISSIONS_REPO_ID and HF_WRITE_TOKEN are configured." submitted_at = dt.datetime.utcnow().replace(microsecond=0).isoformat() + "Z" row = leaderboard_row(team, contact, submission_name, metrics, submitted_at) payload = { "leaderboard": row, "metrics": metrics, "predictions": rows, } timestamp = submitted_at.replace(":", "").replace("-", "") team_slug = safe_slug(team) name_slug = safe_slug(submission_name) filename = f"{timestamp}_{team_slug}_{name_slug}.json" tmp_path = Path("/tmp") / filename tmp_path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") upload_file( path_or_fileobj=str(tmp_path), path_in_repo=f"submissions/{filename}", repo_id=SUBMISSIONS_REPO_ID, repo_type="dataset", token=WRITE_TOKEN, commit_message=f"Add submission {team_slug}/{name_slug}", ) attempts = _update_leaderboard(row) return ( f"Score computed and saved to {SUBMISSIONS_REPO_ID}/submissions/{filename}. " f"This is attempt {attempts} for this team and contact email." ) def _load_leaderboard_rows(): if not SUBMISSIONS_REPO_ID or not WRITE_TOKEN: return [] try: text = _read_hub_file( SUBMISSIONS_REPO_ID, "leaderboard/leaderboard.json", token=WRITE_TOKEN, force_download=True, ) except (EntryNotFoundError, RepositoryNotFoundError): return [] rows = json.loads(text) return rows if isinstance(rows, list) else [] def _sort_leaderboard(rows): return rank_leaderboard(rows) def _update_leaderboard(row): with LEADERBOARD_LOCK: rows = _load_leaderboard_rows() rows.append(row) rows.sort(key=lambda item: str(item.get("submitted_at", ""))) tmp_path = Path("/tmp") / "leaderboard.json" tmp_path.write_text(json.dumps(rows, indent=2, sort_keys=True), encoding="utf-8") upload_file( path_or_fileobj=str(tmp_path), path_in_repo="leaderboard/leaderboard.json", repo_id=SUBMISSIONS_REPO_ID, repo_type="dataset", token=WRITE_TOKEN, commit_message="Update leaderboard", ) identity = leaderboard_identity(row) best = next( item for item in rank_leaderboard(rows) if leaderboard_identity(item) == identity ) return best["attempts"] def _format_metric(value): return f"{float(value) * 100:.2f}%" def _format_timestamp(value): return str(value).replace("T", " ").removesuffix("Z") def leaderboard_html(): rows = _sort_leaderboard(_load_leaderboard_rows()) body_rows = [] for index, row in enumerate(rows[:100], start=1): body_rows.append( f"""
| Rank | Team | Best submission | Attempts | Answer accuracy | Evidence F1 | Best score (UTC) |
|---|
Document-grounded quantitative reasoning with evidence attribution.
Co-located with EMNLP 2026 in Budapest, Hungary. Beyond Plain Text: Bridging NLP and Document AI.
Upload one JSON object per instance with instance_id,
answer, and evidence.
Review the participant guide
for the complete format and evaluation protocol.
Best attempt per team and contact email. Ranked by answer accuracy, then evidence F1. Attempt counts include every valid submission.