code/summarize_sweep.py

#!/usr/bin/env python3
"""Aggregate the S2 backtracking sweep: latest frontier + per-stage eval per combo."""
import glob, os, re, sys

root = sys.argv[1] if len(sys.argv) > 1 else sorted(glob.glob("/home/ubuntu/bt_runs/s2_sweep_*"))[-1]
logs = sorted(glob.glob(os.path.join(root, "logs", "*.log")))

frontier_re = re.compile(r"eval/.*")  # unused; we parse the debug-free summary lines
bt_re = re.compile(r"\[backtrack eval\] stage=(\d+) k=(\d+) exact=([\d.]+) prec=([\d.]+) rec=([\d.]+) solve=([\d.]+)")
rcur_re = re.compile(r"\[backtrack\] r_current=(\d+) min_stage_exact=([\d.]+)")
step_re = re.compile(r"train step (\d+)")
# frontier eval when backtracking is OFF prints via run_eval summary; grab solve/exact if present
summ_re = re.compile(r"exact_set_match_rate[\"']?\s*[:=]\s*([\d.]+)")

print(f"Sweep root: {root}\n")
header = f"{'combo':16} {'step':>5}  {'S1 exact':>8} {'S2 exact':>8} {'S2 prec':>7} {'S2 rec':>7} {'S2 solve':>8}  {'r_cur':>5}"
print(header)
print("-" * len(header))
for lg in logs:
    name = os.path.basename(lg)[:-4]
    txt = open(lg, errors="ignore").read()
    step = step_re.findall(txt)
    laststep = step[-1] if step else "-"
    # collect last per-stage block
    stages = {}
    for m in bt_re.finditer(txt):
        s = int(m.group(1)); stages[s] = tuple(float(x) for x in m.groups()[2:])
    rc = rcur_re.findall(txt)
    if stages:
        s1 = stages.get(1, (float('nan'),)*4)[0]
        s2 = stages.get(2, (float('nan'),)*4)
        rcv = rc[-1][0] if rc else "-"
        print(f"{name:16} {laststep:>5}  {s1:8.3f} {s2[0]:8.3f} {s2[1]:7.3f} {s2[2]:7.3f} {s2[3]:8.3f}  {rcv:>5}")
    else:
        # no-backtrack combos: frontier metrics not in this regex; report step only
        print(f"{name:16} {laststep:>5}  {'(no per-stage eval — control)':>50}")