| """Decoupe un fichier answers_<backend>.json en items individuels pour le juge. |
| |
| Le workflow de jugement (eval/judge_workflow.js) lit un fichier JSON par item. |
| Pour ne plus ecraser les runs entre backends (PLAN 7.5), chaque backend a son |
| repertoire : eval/runs/items_<backend>/<id>.json. |
| |
| Usage : |
| python -m eval.split_items eval/runs/answers_mistral.json |
| -> eval/runs/items_mistral/{concept-01,...}.json |
| Puis : Workflow judge_workflow.js avec args {"itemsDir": ".../runs/items_mistral"}. |
| """ |
| from __future__ import annotations |
|
|
| import json |
| import re |
| import sys |
| from pathlib import Path |
|
|
|
|
| def main() -> int: |
| if len(sys.argv) != 2: |
| print(__doc__) |
| return 1 |
| src = Path(sys.argv[1]) |
| data = json.loads(src.read_text(encoding="utf-8")) |
| m = re.match(r"answers_(.+)\.json$", src.name) |
| backend = m.group(1) if m else src.stem |
| out_dir = src.parent / f"items_{backend}" |
| out_dir.mkdir(parents=True, exist_ok=True) |
| n = 0 |
| for it in data["items"]: |
| keep = {k: it[k] for k in ("id", "category", "prompt", "gold", "answer", |
| "citations", "code_findings", "deterministic") |
| if k in it} |
| (out_dir / f"{it['id']}.json").write_text( |
| json.dumps(keep, ensure_ascii=False, indent=1), encoding="utf-8") |
| n += 1 |
| print(f"{n} items -> {out_dir}") |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
