#!/usr/bin/env python3 """Regenerate the built-in replay fixture for `autocodabench demo --replay`. The fixture is the tool-call sequence a live agent makes when building a small AI-text-detection competition (result-submission style, two phases). It exercises every authoring tool plus validate + zip, and the resulting bundle passes the deterministic check suite — so the demo doubles as an end-to-end self-test of the offline pipeline. Usage: python scripts/make_demo_fixture.py """ from __future__ import annotations import json import random import textwrap from pathlib import Path OUT = Path(__file__).resolve().parents[1] / "src" / "autocodabench" / "backends" / "fixtures" / "demo_bundle.jsonl" SLUG = "demo-ai-text-detection" # 200x200 blue PNG — placeholder competition logo. LOGO_B64 = "iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAIAAAAiOjnJAAABeElEQVR42u3SMQ0AAAgEsfeKCcQiBhMMDE2q4HKpHjgXCTAWxsJYYCyMhbHAWBgLY4GxMBbGAmNhLIwFxsJYGAuMhbEwFhgLY2EsMBbGwlhgLIyFscBYGAtjgbEwFsYCY2EsjAXGwlgYC4yFsTAWGAtjYSwwFsbCWGAsjIWxwFgYC2OBsTAWxgJjYSyMBcbCWBgLjIWxMBYYC2NhLDAWxsJYYCyMhbHAWBgLY4GxMBbGAmNhLIyFsVTAWBgLY4GxMBbGAmNhLIwFxsJYGAuMhbEwFhgLY2EsMBbGwlhgLIyFscBYGAtjgbEwFsYCY2EsjAXGwlgYC4yFsTAWGAtjYSwwFsbCWGAsjIWxwFgYC2OBsTAWxgJjYSyMBcbCWBgLjIWxMBYYC2NhLDAWxsJYYCyMhbHAWBgLY4GxMBbGAmNhLIwFxsJYGAtjqYCxMBbGAmNhLIwFxsJYGAuMhbEwFhgLY2EsMBbGwlhgLIyFscBYGAtjgbH4ZgEv2wF1t+A3nQAAAABJRU5ErkJggg==" SCORE_PY = textwrap.dedent("""\ \"\"\"Codabench scoring program — accuracy + balanced accuracy (stdlib only).\"\"\" import csv import json import os import sys input_dir, output_dir = sys.argv[1], sys.argv[2] def read_labels(path): with open(path, newline="") as f: return [row[0].strip() for row in csv.reader(f) if row] truth = read_labels(os.path.join(input_dir, "ref", "truth.csv")) preds = read_labels(os.path.join(input_dir, "res", "predictions.csv")) if len(preds) != len(truth): sys.exit(f"prediction count {len(preds)} != reference count {len(truth)}") correct = sum(p == t for p, t in zip(preds, truth)) accuracy = correct / len(truth) per_class = {} for p, t in zip(preds, truth): hit, n = per_class.get(t, (0, 0)) per_class[t] = (hit + (p == t), n + 1) balanced_accuracy = sum(h / n for h, n in per_class.values()) / len(per_class) os.makedirs(output_dir, exist_ok=True) with open(os.path.join(output_dir, "scores.json"), "w") as f: json.dump({"accuracy": accuracy, "balanced_accuracy": balanced_accuracy}, f) print(f"accuracy={accuracy:.4f} balanced_accuracy={balanced_accuracy:.4f}") """) OVERVIEW_MD = textwrap.dedent("""\ # AI-Generated Text Detection (demo) Classifying whether a text was written by a human (`0`) or generated by a language model (`1`) is error-prone for humans and high-stakes for platforms. This demo competition asks you to predict the label for each text in the evaluation set. ## Submission format Upload a zip containing a single `predictions.csv` with one label (`0` or `1`) per line, in the same order as the evaluation texts in the starting kit. ## Phases 1. **Development** — public leaderboard, max 5 submissions/day. 2. **Final** — private test set, max 3 submissions total. Rules of thumb in this design follow Pavão et al. (2024), *AI Competitions and Benchmarks*. """) EVALUATION_MD = textwrap.dedent("""\ # Evaluation The primary metric is **balanced accuracy** (the mean of per-class recalls); plain accuracy is reported as a secondary column. Balanced accuracy is used because the class mix is not guaranteed to be 50/50 and accuracy alone rewards the constant predictor (Pavão et al., Ch. 4). Ties on the leaderboard are broken by **earlier submission time**. The scoring program is `scoring_program/score.py` — you can run it locally on the starting-kit data to reproduce the leaderboard number: ```bash python score.py ``` """) DATA_MD = textwrap.dedent("""\ # Data - `starting_kit/` — toy development texts + an example `predictions.csv`. - Reference labels for the leaderboard are sequestered in `reference_data/` and never shipped to participants. The demo dataset is synthetic and CC0-licensed. External data is **not** allowed in this demo competition. """) TERMS_MD = textwrap.dedent("""\ # Terms By participating you agree to: 1. submit predictions produced by your own model or pipeline, 2. not probe the leaderboard with crafted submissions, 3. publish your method (code or report) if you finish in the top 3. Organizers may disqualify submissions that violate these terms. """) STARTING_KIT_README = textwrap.dedent("""\ # Starting kit `dev_texts.csv` holds toy development texts with labels so you can train and sanity-check locally. `predictions_example.csv` shows the exact submission format: one label per line, no header. Quickstart: copy `predictions_example.csv` to `predictions.csv`, zip it, and submit — you should land at the trivial-baseline score within minutes. """) def main() -> None: rng = random.Random(7) n = 40 truth = [str(rng.randint(0, 1)) for _ in range(n)] # Baseline predictions: ~80% correct — clearly above the trivial baseline. preds = [t if rng.random() < 0.8 else str(1 - int(t)) for t in truth] dev_texts = "\n".join( f"text_{i:02d},\"{'A human wrote this short note.' if t == '0' else 'As an AI language model I produced this.'}\",{t}" for i, t in enumerate(truth) ) facts_yaml = textwrap.dedent("""\ # Declared competition facts — consumed by `autocodabench validate`. # See autocodabench.checks.facts for the schema. anticipated_error_rate: 0.2 # toy demo — the 100/E check will flag the tiny test set unit_of_generalization: document external_data_allowed: false prizes: false task_type: binary_classification """) records: list[dict] = [ {"tool": "autocodabench_init_bundle", "args": {"slug": SLUG, "overwrite": True}}, {"tool": "_write_file", "args": {"slug": SLUG, "path": "logo.png", "base64": LOGO_B64}}, {"tool": "_write_file", "args": {"slug": SLUG, "path": "competition_facts.yaml", "text": facts_yaml}}, {"tool": "autocodabench_write_page", "args": {"slug": SLUG, "filename": "overview.md", "body": OVERVIEW_MD}}, {"tool": "autocodabench_write_page", "args": {"slug": SLUG, "filename": "evaluation.md", "body": EVALUATION_MD}}, {"tool": "autocodabench_write_page", "args": {"slug": SLUG, "filename": "data.md", "body": DATA_MD}}, {"tool": "autocodabench_write_page", "args": {"slug": SLUG, "filename": "terms.md", "body": TERMS_MD}}, {"tool": "autocodabench_write_scoring_program", "args": {"slug": SLUG, "script": SCORE_PY}}, {"tool": "autocodabench_attach_data", "args": { "slug": SLUG, "target": "reference_data", "files": {"truth.csv": "\n".join(truth) + "\n"}, }}, {"tool": "autocodabench_attach_data", "args": { "slug": SLUG, "target": "starting_kit", "files": { "README.md": STARTING_KIT_README, "dev_texts.csv": "id,text,label\n" + dev_texts + "\n", "predictions_example.csv": "\n".join(preds) + "\n", }, }}, {"tool": "autocodabench_write_solution", "args": { "slug": SLUG, "files": {"predictions.csv": "\n".join(preds) + "\n"}, }}, {"tool": "autocodabench_write_competition_yaml", "args": {"slug": SLUG, "payload": { "version": 2, "title": "AI-Generated Text Detection (autocodabench demo)", "description": "Detect AI-generated text. Built offline by the autocodabench replay demo.", "image": "logo.png", "terms": "pages/terms.md", "registration_auto_approve": True, "docker_image": "codalab/codalab-legacy:py39", "pages": [ {"title": "Overview", "file": "pages/overview.md"}, {"title": "Evaluation", "file": "pages/evaluation.md"}, {"title": "Data", "file": "pages/data.md"}, {"title": "Terms", "file": "pages/terms.md"}, ], "tasks": [{ "index": 0, "name": "AI-text detection", "description": "Binary classification of human vs AI text.", "scoring_program": "scoring_program/", "reference_data": "reference_data/", }], "solutions": [{"index": 0, "path": "solutions/solution_baseline/", "tasks": [0]}], "phases": [ { "index": 0, "name": "Development", "description": "Public leaderboard on development labels.", "start": "2026-07-01 00:00:00", "end": "2026-08-15 00:00:00", "max_submissions_per_day": 5, "max_submissions": 100, "tasks": [0], }, { "index": 1, "name": "Final", "description": "Private test set; 3 submissions total.", "start": "2026-08-15 00:00:00", "end": "2026-08-22 00:00:00", "max_submissions": 3, "tasks": [0], }, ], "leaderboards": [{ "title": "Results", "key": "main", "columns": [ {"title": "Balanced accuracy", "key": "balanced_accuracy", "index": 0, "sorting": "desc"}, {"title": "Accuracy", "key": "accuracy", "index": 1, "sorting": "desc"}, ], }], }}}, {"tool": "autocodabench_validate_bundle", "args": {"slug": SLUG}}, {"tool": "autocodabench_zip_bundle", "args": {"slug": SLUG}}, {"final_text": ( "Built the demo competition bundle **demo-ai-text-detection** from a " "recorded agent run: 4 pages, a stdlib-only scoring program, a " "baseline solution, sequestered reference labels, and a two-phase " "schedule (development with daily caps → final with 3 submissions). " "The bundle was validated and zipped — fully offline, no API keys." )}, ] OUT.parent.mkdir(parents=True, exist_ok=True) with OUT.open("w", encoding="utf-8") as f: for rec in records: f.write(json.dumps(rec, ensure_ascii=False) + "\n") print(f"wrote {OUT} ({len(records)} records)") if __name__ == "__main__": main()