Buckets:
| """List the ProgramBench tasks with their metadata and select a balanced 30-task subset. | |
| UV_PROJECT_ENVIRONMENT=<swarmlab env> uv run --project <swarmlab checkout> \ | |
| python scripts/programbench_subset.py [--out-dir tasks] [--check] | |
| (from experiments/flint-programbench/; `--out-dir` defaults to `tasks/` next to this script's | |
| directory). Needs the `programbench` package (`swarmlab[programbench]`); reads only the task metadata it | |
| ships (`programbench.utils.load_data.load_all_instances`: each task's `task.yaml` and | |
| `tests.json`), downloads nothing. | |
| The table (`programbench-tasks.csv`) has one row per task: `instance_id`, `repository`, `language` | |
| (of the reference program), `difficulty` (the benchmark's own label: easy, medium, hard, or | |
| empty when the task has none), `active_branches` (test branches not ignored), `tests` (the | |
| size proxy: the hidden tests that count, i.e. the tests of the active branches minus their | |
| ignored tests), `ignored_tests`, and `selected` (the stratum when the task is in the subset). | |
| The selection rule (`select`), deterministic, with no randomness: | |
| 1. The pool: the 200 benchmark tasks (the package's `testorg__calculator.abc1234`, a | |
| 3-test fixture, is left out) that carry a difficulty label (165 of them; 35 have none). | |
| 2. Strata: the label, 10 easy, 10 medium, 10 hard, as the paper's "Balanced-30" (arXiv | |
| 2610.17471, App C.2: 10 easy, 10 medium, 10 hard; its ids are not published). | |
| 3. Within a stratum, the 10 slots are spread over the reference languages: every language of | |
| the stratum gets one slot (in order of its task count, then name, while slots remain), the rest | |
| go by largest remainder of the language's share of the stratum. | |
| 4. Within a language with k slots, its tasks are sorted by `tests` (then id), and the tasks at | |
| the quantile positions floor((j + 0.5) * n / k), j = 0..k-1, are taken: a spread from small to | |
| large test suites, not the k smallest. | |
| `--check` exits 1 when the files in `--out-dir` differ from what the script would write. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import io | |
| import sys | |
| from collections import Counter | |
| from pathlib import Path | |
| FIXTURES = {"testorg__calculator.abc1234"} | |
| STRATA = ("easy", "medium", "hard") | |
| PER_STRATUM = 10 | |
| COLUMNS = ("instance_id", "repository", "language", "difficulty", "active_branches", "tests", | |
| "ignored_tests", "selected") | |
| LIST_HEADER = """\ | |
| # ProgramBench Balanced-30 for the replication of arXiv 2610.17471 Sec 4 (solo PIV vs team PIV). | |
| # Written by scripts/programbench_subset.py (programbench {version}); the rule is in its docstring: | |
| # the 200 tasks with a difficulty label, 10 easy / 10 medium / 10 hard (the benchmark's label), | |
| # each stratum's slots spread over the reference languages (one per language, the rest by share), | |
| # and within a language a spread over the hidden-test count (quantile positions, no randomness). | |
| # The metadata of all 200 tasks is in programbench-tasks.csv next to this file. | |
| # One task id per line, in stratum order (easy, medium, hard), then by id. | |
| """ | |
| def task_rows() -> list[dict]: | |
| """One metadata row per benchmark task (fixtures left out), sorted by id.""" | |
| from programbench.utils.load_data import get_active_branches, load_all_instances | |
| rows = [] | |
| for inst in load_all_instances(): | |
| if inst["instance_id"] in FIXTURES: | |
| continue | |
| active = get_active_branches(inst) | |
| branches = inst.get("branches") or {} | |
| ignored = sum(len(branches[b].get("ignored_tests") or []) for b in active) | |
| total = sum(len(branches[b].get("tests") or []) for b in active) | |
| rows.append({"instance_id": inst["instance_id"], "repository": inst["repository"], | |
| "language": inst["language"], "difficulty": inst.get("difficulty") or "", | |
| "active_branches": len(active), "tests": total - ignored, | |
| "ignored_tests": ignored, "selected": ""}) | |
| return sorted(rows, key=lambda r: r["instance_id"]) | |
| def language_slots(counts: Counter, slots: int) -> dict[str, int]: | |
| """`slots` spread over languages: one each (largest first), then by largest remainder.""" | |
| langs = sorted(counts, key=lambda lang: (-counts[lang], lang)) | |
| out = {lang: 0 for lang in langs} | |
| for lang in langs[:slots]: | |
| out[lang] = 1 | |
| left = slots - sum(out.values()) | |
| if left > 0: | |
| total = sum(counts.values()) | |
| quota = {lang: left * counts[lang] / total for lang in langs} | |
| for lang in langs: | |
| out[lang] += int(quota[lang]) | |
| rest = slots - sum(out.values()) | |
| for lang in sorted(langs, key=lambda lang: (-(quota[lang] % 1), -counts[lang], lang))[:rest]: | |
| out[lang] += 1 | |
| return {lang: k for lang, k in out.items() if k} | |
| def spread(rows: list[dict], k: int) -> list[dict]: | |
| """k rows at the quantile positions of `rows` sorted by test count (then id).""" | |
| ordered = sorted(rows, key=lambda r: (r["tests"], r["instance_id"])) | |
| n = len(ordered) | |
| return [ordered[int((j + 0.5) * n / k)] for j in range(min(k, n))] | |
| def select(rows: list[dict]) -> dict[str, list[dict]]: | |
| """The balanced subset, `{stratum: rows}` (module doc).""" | |
| chosen = {} | |
| for stratum in STRATA: | |
| pool = [r for r in rows if r["difficulty"] == stratum] | |
| slots = language_slots(Counter(r["language"] for r in pool), PER_STRATUM) | |
| picked = [] | |
| for lang, k in slots.items(): | |
| picked += spread([r for r in pool if r["language"] == lang], k) | |
| chosen[stratum] = sorted(picked, key=lambda r: r["instance_id"]) | |
| return chosen | |
| def render(rows: list[dict], chosen: dict[str, list[dict]], version: str) -> tuple[str, str]: | |
| """(the task list, the metadata CSV) as written to the output directory.""" | |
| stratum_of = {r["instance_id"]: s for s, rs in chosen.items() for r in rs} | |
| buf = io.StringIO() | |
| writer = csv.DictWriter(buf, fieldnames=COLUMNS, lineterminator="\n") | |
| writer.writeheader() | |
| for r in rows: | |
| writer.writerow({**r, "selected": stratum_of.get(r["instance_id"], "")}) | |
| ids = [r["instance_id"] for s in STRATA for r in chosen[s]] | |
| return LIST_HEADER.format(version=version) + "\n".join(ids) + "\n", buf.getvalue() | |
| def main(argv: list[str] | None = None) -> int: | |
| ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) | |
| ap.add_argument("--out-dir", default=Path(__file__).resolve().parents[1] / "tasks", type=Path) | |
| ap.add_argument("--check", action="store_true", | |
| help="exit 1 if the files in --out-dir differ from the selection") | |
| args = ap.parse_args(argv) | |
| from importlib.metadata import version | |
| rows = task_rows() | |
| chosen = select(rows) | |
| listing, table = render(rows, chosen, version("programbench")) | |
| files = {args.out_dir / "tasks-balanced-30.txt": listing, | |
| args.out_dir / "programbench-tasks.csv": table} | |
| print(f"{len(rows)} tasks; by difficulty: " | |
| + ", ".join(f"{k or 'none'} {v}" for k, v in | |
| sorted(Counter(r['difficulty'] for r in rows).items()))) | |
| print(f"{'stratum':8} {'instance_id':48} {'lang':5} {'tests':>6}") | |
| for s in STRATA: | |
| for r in chosen[s]: | |
| print(f"{s:8} {r['instance_id']:48} {r['language']:5} {r['tests']:>6}") | |
| langs = Counter(r["language"] for r in chosen[s]) | |
| print(f"{'':8} languages: {dict(sorted(langs.items()))}") | |
| if args.check: | |
| stale = [str(p) for p, text in files.items() | |
| if not p.exists() or p.read_text() != text] | |
| if stale: | |
| print("out of date: " + ", ".join(stale), file=sys.stderr) | |
| return 1 | |
| print("up to date") | |
| return 0 | |
| args.out_dir.mkdir(parents=True, exist_ok=True) | |
| for path, text in files.items(): | |
| path.write_text(text) | |
| print(f"wrote {path}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |
Xet Storage Details
- Size:
- 7.95 kB
- Xet hash:
- dd0c127d6ec4d40f67b3f6e0732051cda80915616c3686992142d7ff62826b55
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.