scripts/make_demo_fixture.py

#!/usr/bin/env python3
"""Regenerate the built-in replay fixture for `autocodabench demo --replay`.

The fixture is the tool-call sequence a live agent makes when building a
small AI-text-detection competition (result-submission style, two phases).
It exercises every authoring tool plus validate + zip, and the resulting
bundle passes the deterministic check suite — so the demo doubles as an
end-to-end self-test of the offline pipeline.

Usage: python scripts/make_demo_fixture.py
"""
from __future__ import annotations

import json
import random
import textwrap
from pathlib import Path

OUT = Path(__file__).resolve().parents[1] / "src" / "autocodabench" / "backends" / "fixtures" / "demo_bundle.jsonl"

SLUG = "demo-ai-text-detection"

# 200x200 blue PNG — placeholder competition logo.
LOGO_B64 = "iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAIAAAAiOjnJAAABeElEQVR42u3SMQ0AAAgEsfeKCcQiBhMMDE2q4HKpHjgXCTAWxsJYYCyMhbHAWBgLY4GxMBbGAmNhLIwFxsJYGAuMhbEwFhgLY2EsMBbGwlhgLIyFscBYGAtjgbEwFsYCY2EsjAXGwlgYC4yFsTAWGAtjYSwwFsbCWGAsjIWxwFgYC2OBsTAWxgJjYSyMBcbCWBgLjIWxMBYYC2NhLDAWxsJYYCyMhbHAWBgLY4GxMBbGAmNhLIyFsVTAWBgLY4GxMBbGAmNhLIwFxsJYGAuMhbEwFhgLY2EsMBbGwlhgLIyFscBYGAtjgbEwFsYCY2EsjAXGwlgYC4yFsTAWGAtjYSwwFsbCWGAsjIWxwFgYC2OBsTAWxgJjYSyMBcbCWBgLjIWxMBYYC2NhLDAWxsJYYCyMhbHAWBgLY4GxMBbGAmNhLIwFxsJYGAtjqYCxMBbGAmNhLIwFxsJYGAuMhbEwFhgLY2EsMBbGwlhgLIyFscBYGAtjgbH4ZgEv2wF1t+A3nQAAAABJRU5ErkJggg=="

SCORE_PY = textwrap.dedent("""\
    \"\"\"Codabench scoring program — accuracy + balanced accuracy (stdlib only).\"\"\"
    import csv
    import json
    import os
    import sys

    input_dir, output_dir = sys.argv[1], sys.argv[2]

    def read_labels(path):
        with open(path, newline="") as f:
            return [row[0].strip() for row in csv.reader(f) if row]

    truth = read_labels(os.path.join(input_dir, "ref", "truth.csv"))
    preds = read_labels(os.path.join(input_dir, "res", "predictions.csv"))

    if len(preds) != len(truth):
        sys.exit(f"prediction count {len(preds)} != reference count {len(truth)}")

    correct = sum(p == t for p, t in zip(preds, truth))
    accuracy = correct / len(truth)

    per_class = {}
    for p, t in zip(preds, truth):
        hit, n = per_class.get(t, (0, 0))
        per_class[t] = (hit + (p == t), n + 1)
    balanced_accuracy = sum(h / n for h, n in per_class.values()) / len(per_class)

    os.makedirs(output_dir, exist_ok=True)
    with open(os.path.join(output_dir, "scores.json"), "w") as f:
        json.dump({"accuracy": accuracy, "balanced_accuracy": balanced_accuracy}, f)
    print(f"accuracy={accuracy:.4f} balanced_accuracy={balanced_accuracy:.4f}")
""")

OVERVIEW_MD = textwrap.dedent("""\
    # AI-Generated Text Detection (demo)

    Classifying whether a text was written by a human (`0`) or generated by a
    language model (`1`) is error-prone for humans and high-stakes for
    platforms. This demo competition asks you to predict the label for each
    text in the evaluation set.

    ## Submission format

    Upload a zip containing a single `predictions.csv` with one label
    (`0` or `1`) per line, in the same order as the evaluation texts in the
    starting kit.

    ## Phases

    1. **Development** — public leaderboard, max 5 submissions/day.
    2. **Final** — private test set, max 3 submissions total.

    Rules of thumb in this design follow Pavão et al. (2024),
    *AI Competitions and Benchmarks*.
""")

EVALUATION_MD = textwrap.dedent("""\
    # Evaluation

    The primary metric is **balanced accuracy** (the mean of per-class
    recalls); plain accuracy is reported as a secondary column. Balanced
    accuracy is used because the class mix is not guaranteed to be 50/50 and
    accuracy alone rewards the constant predictor (Pavão et al., Ch. 4).

    Ties on the leaderboard are broken by **earlier submission time**.

    The scoring program is `scoring_program/score.py` — you can run it
    locally on the starting-kit data to reproduce the leaderboard number:

    ```bash
    python score.py <input_dir> <output_dir>
    ```
""")

DATA_MD = textwrap.dedent("""\
    # Data

    - `starting_kit/` — toy development texts + an example `predictions.csv`.
    - Reference labels for the leaderboard are sequestered in
      `reference_data/` and never shipped to participants.

    The demo dataset is synthetic and CC0-licensed. External data is **not**
    allowed in this demo competition.
""")

TERMS_MD = textwrap.dedent("""\
    # Terms

    By participating you agree to:

    1. submit predictions produced by your own model or pipeline,
    2. not probe the leaderboard with crafted submissions,
    3. publish your method (code or report) if you finish in the top 3.

    Organizers may disqualify submissions that violate these terms.
""")

STARTING_KIT_README = textwrap.dedent("""\
    # Starting kit

    `dev_texts.csv` holds toy development texts with labels so you can train
    and sanity-check locally. `predictions_example.csv` shows the exact
    submission format: one label per line, no header.

    Quickstart: copy `predictions_example.csv` to `predictions.csv`, zip it,
    and submit — you should land at the trivial-baseline score within
    minutes.
""")


def main() -> None:
    rng = random.Random(7)
    n = 40
    truth = [str(rng.randint(0, 1)) for _ in range(n)]
    # Baseline predictions: ~80% correct — clearly above the trivial baseline.
    preds = [t if rng.random() < 0.8 else str(1 - int(t)) for t in truth]
    dev_texts = "\n".join(
        f"text_{i:02d},\"{'A human wrote this short note.' if t == '0' else 'As an AI language model I produced this.'}\",{t}"
        for i, t in enumerate(truth)
    )

    facts_yaml = textwrap.dedent("""\
        # Declared competition facts — consumed by `autocodabench validate`.
        # See autocodabench.checks.facts for the schema.
        anticipated_error_rate: 0.2   # toy demo — the 100/E check will flag the tiny test set
        unit_of_generalization: document
        external_data_allowed: false
        prizes: false
        task_type: binary_classification
    """)

    records: list[dict] = [
        {"tool": "autocodabench_init_bundle", "args": {"slug": SLUG, "overwrite": True}},
        {"tool": "_write_file", "args": {"slug": SLUG, "path": "logo.png", "base64": LOGO_B64}},
        {"tool": "_write_file", "args": {"slug": SLUG, "path": "competition_facts.yaml", "text": facts_yaml}},
        {"tool": "autocodabench_write_page", "args": {"slug": SLUG, "filename": "overview.md", "body": OVERVIEW_MD}},
        {"tool": "autocodabench_write_page", "args": {"slug": SLUG, "filename": "evaluation.md", "body": EVALUATION_MD}},
        {"tool": "autocodabench_write_page", "args": {"slug": SLUG, "filename": "data.md", "body": DATA_MD}},
        {"tool": "autocodabench_write_page", "args": {"slug": SLUG, "filename": "terms.md", "body": TERMS_MD}},
        {"tool": "autocodabench_write_scoring_program", "args": {"slug": SLUG, "script": SCORE_PY}},
        {"tool": "autocodabench_attach_data", "args": {
            "slug": SLUG, "target": "reference_data",
            "files": {"truth.csv": "\n".join(truth) + "\n"},
        }},
        {"tool": "autocodabench_attach_data", "args": {
            "slug": SLUG, "target": "starting_kit",
            "files": {
                "README.md": STARTING_KIT_README,
                "dev_texts.csv": "id,text,label\n" + dev_texts + "\n",
                "predictions_example.csv": "\n".join(preds) + "\n",
            },
        }},
        {"tool": "autocodabench_write_solution", "args": {
            "slug": SLUG,
            "files": {"predictions.csv": "\n".join(preds) + "\n"},
        }},
        {"tool": "autocodabench_write_competition_yaml", "args": {"slug": SLUG, "payload": {
            "version": 2,
            "title": "AI-Generated Text Detection (autocodabench demo)",
            "description": "Detect AI-generated text. Built offline by the autocodabench replay demo.",
            "image": "logo.png",
            "terms": "pages/terms.md",
            "registration_auto_approve": True,
            "docker_image": "codalab/codalab-legacy:py39",
            "pages": [
                {"title": "Overview", "file": "pages/overview.md"},
                {"title": "Evaluation", "file": "pages/evaluation.md"},
                {"title": "Data", "file": "pages/data.md"},
                {"title": "Terms", "file": "pages/terms.md"},
            ],
            "tasks": [{
                "index": 0,
                "name": "AI-text detection",
                "description": "Binary classification of human vs AI text.",
                "scoring_program": "scoring_program/",
                "reference_data": "reference_data/",
            }],
            "solutions": [{"index": 0, "path": "solutions/solution_baseline/", "tasks": [0]}],
            "phases": [
                {
                    "index": 0,
                    "name": "Development",
                    "description": "Public leaderboard on development labels.",
                    "start": "2026-07-01 00:00:00",
                    "end": "2026-08-15 00:00:00",
                    "max_submissions_per_day": 5,
                    "max_submissions": 100,
                    "tasks": [0],
                },
                {
                    "index": 1,
                    "name": "Final",
                    "description": "Private test set; 3 submissions total.",
                    "start": "2026-08-15 00:00:00",
                    "end": "2026-08-22 00:00:00",
                    "max_submissions": 3,
                    "tasks": [0],
                },
            ],
            "leaderboards": [{
                "title": "Results",
                "key": "main",
                "columns": [
                    {"title": "Balanced accuracy", "key": "balanced_accuracy", "index": 0, "sorting": "desc"},
                    {"title": "Accuracy", "key": "accuracy", "index": 1, "sorting": "desc"},
                ],
            }],
        }}},
        {"tool": "autocodabench_validate_bundle", "args": {"slug": SLUG}},
        {"tool": "autocodabench_zip_bundle", "args": {"slug": SLUG}},
        {"final_text": (
            "Built the demo competition bundle **demo-ai-text-detection** from a "
            "recorded agent run: 4 pages, a stdlib-only scoring program, a "
            "baseline solution, sequestered reference labels, and a two-phase "
            "schedule (development with daily caps → final with 3 submissions). "
            "The bundle was validated and zipped — fully offline, no API keys."
        )},
    ]

    OUT.parent.mkdir(parents=True, exist_ok=True)
    with OUT.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec, ensure_ascii=False) + "\n")
    print(f"wrote {OUT} ({len(records)} records)")


if __name__ == "__main__":
    main()