File size: 1,431 Bytes
5043c6f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | """Decoupe un fichier answers_<backend>.json en items individuels pour le juge.
Le workflow de jugement (eval/judge_workflow.js) lit un fichier JSON par item.
Pour ne plus ecraser les runs entre backends (PLAN 7.5), chaque backend a son
repertoire : eval/runs/items_<backend>/<id>.json.
Usage :
python -m eval.split_items eval/runs/answers_mistral.json
-> eval/runs/items_mistral/{concept-01,...}.json
Puis : Workflow judge_workflow.js avec args {"itemsDir": ".../runs/items_mistral"}.
"""
from __future__ import annotations
import json
import re
import sys
from pathlib import Path
def main() -> int:
if len(sys.argv) != 2:
print(__doc__)
return 1
src = Path(sys.argv[1])
data = json.loads(src.read_text(encoding="utf-8"))
m = re.match(r"answers_(.+)\.json$", src.name)
backend = m.group(1) if m else src.stem
out_dir = src.parent / f"items_{backend}"
out_dir.mkdir(parents=True, exist_ok=True)
n = 0
for it in data["items"]:
keep = {k: it[k] for k in ("id", "category", "prompt", "gold", "answer",
"citations", "code_findings", "deterministic")
if k in it}
(out_dir / f"{it['id']}.json").write_text(
json.dumps(keep, ensure_ascii=False, indent=1), encoding="utf-8")
n += 1
print(f"{n} items -> {out_dir}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|