File size: 901 Bytes
b2ebc95 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | #!/usr/bin/env python3
"""Count graded episodes in a run dir. `eval --resume` appends to the same traces.jsonl, so
log-line counts undercount across resumes — this is the number that actually matters."""
import json, os, sys
# `-n` prints just the graded count, one per line. Needed because a naive
# `grep -o 'graded= *[0-9]*'` also matches the "ungraded=" field and returns two numbers.
QUIET = "-n" in sys.argv
for d in [a for a in sys.argv[1:] if a != "-n"]:
p = os.path.join(d, "traces.jsonl")
graded = ungraded = 0
if os.path.exists(p):
for line in open(p):
for t in json.loads(line).get("traces", []):
if t.get("rewards"):
graded += 1
else:
ungraded += 1
if QUIET:
print(graded)
else:
print(f"{os.path.basename(d):18s} graded={graded:4d} ungraded={ungraded:4d}")
|