Image Segmentation
PyTorch
English
painting-vision-robotics-kit
semantic-segmentation
robotics
edge-ai
construction-ai
autonomous-painting
wall-painting-robot
paint-coverage-estimation
building-facade
drywall
skirting-detection
window-detection
lidar
depth-validation
segformer
deeplabv3
Eval Results (legacy)
Update code: real-machine G-code adapter, commissioning gate, machine_job runner, pump watchdog firmware, twin completion gate, realism and masking modules (card and weights unchanged)
4eecd03 verified Download benchmark_robotics.py from constructelligence/painting-vision-robotics-kit: direct link, hf CLI and curl.
- Browser
- Download file 22.9 kB
-
https://huggingface.co/constructelligence/painting-vision-robotics-kit/resolve/main/benchmark_robotics.py
- Command line
-
hf download hf://constructelligence/painting-vision-robotics-kit/benchmark_robotics.py
-
curl -L -o benchmark_robotics.py https://huggingface.co/constructelligence/painting-vision-robotics-kit/resolve/main/benchmark_robotics.py
22.9 kB
| #!/usr/bin/env python3 | |
| """A fixed protocol for the robotics layer: same scenarios, same seeds, every run. | |
| `benchmark/SPEC.md` does this for perception -- one protocol, one pinned dataset, one emitted record a | |
| stranger can re-derive. The robotics half (`coverage_loop.py`, `mission.py`, `safety_supervisor.py`) has | |
| had no equivalent: its numbers lived in prose, hand-run, in a doc. That is what the roadmap's standing | |
| requirements forbid (*an unlogged result cannot be cited*). This is the missing protocol. | |
| **What is measured.** Every scenario is the same 2.7 m wall plane on the same three seeds, and each one | |
| isolates a single stressor -- a refused room, an unreadable patch, a biased or systematically wrong | |
| re-scan, undelivered passes, a controller that stops proving it is alive. The metrics are the ones that | |
| can be wrong in a way that matters: | |
| * ``false_complete`` -- the loop said finished and the wall was *not*. The only unforgivable metric. | |
| * ``gap`` -- |believed - true| coverage. How far the machine's picture is from the wall. | |
| * ``paint_l`` / ``minutes`` / ``rounds`` -- what the verdict cost. | |
| * ``outcome`` -- must be one of the **named** outcomes, every time. A crash or a hang is a failure. | |
| * job-level: litres spent on a wall the gate refused (**must be 0**), what a resume saves, and whether a | |
| latched safe-stop stops the *order* and survives the process. | |
| **The floor.** An open-loop controller -- plan once, paint once, declare the wall done, never re-observe | |
| -- is reported alongside. It is cheap and it always reports success, so it can only be judged on | |
| ``false_complete``; it fails that gate, which is the whole justification for a closed loop existing. | |
| That mirrors ``floor:all-other`` in the perception benchmark, and it is what the retired | |
| ``hardware.closed_loop`` demo produced. | |
| **Honest scope.** The observation is *simulated* with the estimator error measured elsewhere | |
| (`docs/HARDWARE.md`), not taken from a camera on a machine. This benchmark gates the control and safety | |
| logic, not the perception it consumes, and the emitted record says so in ``scope``. | |
| python3 benchmark_robotics.py --out benchmark/records/robotics_1.json | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| import coverage_loop as cl | |
| import hardware as hw | |
| from mission import JobWall, Step, reset_latch, run_job | |
| from robot_planner import build_plan | |
| from safety_supervisor import SafeStop | |
| SUITE_VERSION = "robotics/1.0" | |
| MM_PER_UNIT = 10.0 | |
| SEEDS = (0, 1, 2) # every loop scenario runs on all three; job scenarios on the first | |
| JOB_SEEDS = (0,) | |
| PLATFORM = "mast_arm" | |
| APPROVED = {"decision": "approve"} | |
| REJECTED = {"decision": "reject", "reasons": ["no operator approval for this room"]} | |
| SCOPE = ("software: the observation is simulated with the measured estimator error; no camera, no machine") | |
| # One stressor per scenario, so a red gate names the thing that broke. `budget` is merged into Budget, | |
| # `reads` overrides it, and `stall_after` is a controller that goes quiet (with a finite watchdog). | |
| SCENARIOS = ( | |
| {"name": "clean_3reads", "reads": 3, "budget": {}}, | |
| {"name": "clean_1read", "reads": 1, "budget": {}}, | |
| {"name": "biased_1read", "reads": 1, "budget": {"under_claim": 0.05}}, | |
| {"name": "biased_3reads", "reads": 3, "budget": {"under_claim": 0.05}}, | |
| {"name": "systematic_bias_3reads", "reads": 3, "budget": {"under_claim": 0.05, "systematic": True}}, | |
| {"name": "overclaim_3reads", "reads": 3, "budget": {"over_claim": 0.3}}, | |
| {"name": "gate_refused", "reads": 3, "budget": {}, "decision": REJECTED}, | |
| {"name": "ambiguous_gate_on", "reads": 3, "budget": {"ambiguous_patch": 0.15}}, | |
| {"name": "ambiguous_gate_off", "reads": 3, "budget": {"ambiguous_patch": 0.15, "max_ambiguous": 1.0}}, | |
| {"name": "undelivered_passes", "reads": 3, "budget": {"miss_rate": 0.5}}, | |
| {"name": "target_above_geometry", "reads": 3, "budget": {"target": 0.999}}, | |
| {"name": "nothing_to_paint", "reads": 3, "budget": {}, "empty": True}, | |
| {"name": "stale_controller", "reads": 3, "budget": {}, "stall_after": 2}, | |
| ) | |
| NAMED_OUTCOMES = ("covered", "stalled", "budget_exhausted", "time_exhausted", "rounds_exhausted", | |
| "nothing_left_to_plan", "ambiguous", "refused", "safety_stop", "nothing_to_paint") | |
| def fixture(empty=False): | |
| paintable, keepout = cl.wall() | |
| if empty: | |
| paintable = np.zeros_like(paintable) | |
| return paintable, keepout | |
| def supervisor_for(paintable, limits): | |
| return hw.supervisor_for(paintable.shape[1] * MM_PER_UNIT, paintable.shape[0] * MM_PER_UNIT, | |
| PLATFORM, limits=limits) | |
| def one_round_of_paint(paintable, keepout, miss_rate, seed, operation="paint_roller", | |
| mm_per_unit=MM_PER_UNIT): | |
| """What a single open-loop pass delivers, with no re-observation at all. The floor's engine.""" | |
| tool, brush = hw.TOOLS[hw.OPERATIONS[operation]], hw.TOOLS["brush"] | |
| target = np.asarray(paintable, bool) & ~np.asarray(keepout, bool) | |
| height_mm = paintable.shape[0] * mm_per_unit | |
| plan, _ = build_plan(paintable, keepout, stroke_width=max(2, int(tool.width_mm / mm_per_unit)), | |
| overlap=tool.overlap, margin=0, orientation="horizontal", corners=None, | |
| mm_per_unit=mm_per_unit, boundary_tolerance=2.0, | |
| already_painted=np.zeros_like(target), | |
| trim_width=max(2, int(brush.width_mm / mm_per_unit)), | |
| trim_overlap=brush.overlap) | |
| strokes, trim = hw.strokes_from_plan(plan)[0], hw.strokes_from_plan(plan, trim=True)[0] | |
| delivered, minutes = np.zeros_like(target), 0.0 | |
| rng = np.random.default_rng(seed) | |
| for stroke_set, stroke_op in ((strokes, operation), (trim, "cut_in")): | |
| if not stroke_set: | |
| continue | |
| job = hw.compile_job(stroke_set, height_mm, stroke_op, PLATFORM, wall_id="floor") | |
| missed = set(np.flatnonzero(rng.random(len(job["commands"])) < miss_rate)) | |
| delivered |= hw.rasterize(job, target.shape, mm_per_unit, height_mm, | |
| miss=lambda i, missed=missed: i in missed) & target | |
| minutes += job["totals"]["duration_min"] | |
| volume = cl.litres(delivered, mm_per_unit, cl.Budget()) | |
| return {"coverage_true": float((delivered & target).sum() / target.sum()), | |
| "paint_l": round(volume, 3), "minutes": round(minutes, 1)} | |
| def run_scenario(scenario, seed): | |
| """One scenario, one seed: the loop's verdict plus what it cost and what the verdict was worth.""" | |
| paintable, keepout = fixture(scenario.get("empty", False)) | |
| budget = cl.Budget(reads=scenario.get("reads", 3), **scenario.get("budget", {})) | |
| supervisor = None | |
| if scenario.get("stall_after") is not None: | |
| supervisor = supervisor_for(paintable, {"heartbeat_timeout_s": 1e-9}) | |
| result = cl.run(paintable, keepout, MM_PER_UNIT, scenario.get("decision", APPROVED), budget, | |
| operation=scenario.get("operation", "paint_roller"), platform=PLATFORM, | |
| wall_id=scenario["name"], seed=seed, supervisor=supervisor, | |
| stall_after=scenario.get("stall_after")) | |
| return {"outcome": result["outcome"], "rounds": len(result["rounds"]), | |
| "coverage_true": result["coverage_true"], "coverage_believed": result["coverage_observed"], | |
| "gap": round(abs(result["coverage_observed"] - result["coverage_true"]), 4), | |
| "false_complete": result["false_complete"], "latched": result["latched"], | |
| "paint_l": result["paint_l"], "minutes": result["minutes"], | |
| "reason": result["reason"]} | |
| def run_loop_scenarios(seeds=SEEDS): | |
| """Every scenario on every seed, reduced to a mean and an outcome set (unstable outcomes are named).""" | |
| rows, silent, unnamed = [], 0, 0 | |
| for scenario in SCENARIOS: | |
| runs = [run_scenario(scenario, seed) for seed in seeds] | |
| outcomes = sorted({r["outcome"] for r in runs}) | |
| silent += sum(1 for r in runs if r["false_complete"]) | |
| unnamed += sum(1 for r in runs if r["outcome"] not in NAMED_OUTCOMES) | |
| mean = lambda key: round(float(np.mean([r[key] for r in runs])), 4) # noqa: E731 | |
| rows.append({"name": scenario["name"], "outcomes": outcomes, | |
| "outcome_stable": len(outcomes) == 1, "runs": len(runs), | |
| "gap": mean("gap"), "rounds": mean("rounds"), "paint_l": mean("paint_l"), | |
| "minutes": mean("minutes"), "coverage_true": mean("coverage_true"), | |
| "coverage_believed": mean("coverage_believed"), | |
| "false_complete": sum(1 for r in runs if r["false_complete"]), | |
| "reason": runs[0]["reason"]}) | |
| return rows, silent, unnamed | |
| def job_walls(steps, unreadable=True): | |
| """The order the layer is built for: one clean wall, one the gate refuses, one it cannot read.""" | |
| paintable, keepout = fixture() | |
| return [JobWall("wall_1", paintable.copy(), keepout.copy(), steps, dict(APPROVED), | |
| {"under_claim": 0.05}), | |
| JobWall("wall_2", paintable.copy(), keepout.copy(), steps, dict(REJECTED), {}), | |
| JobWall("wall_3", paintable.copy(), keepout.copy(), steps, dict(APPROVED), | |
| {"ambiguous_patch": 0.15} if unreadable else {"under_claim": 0.05})] | |
| def run_job_probe(tmp_path, seeds=JOB_SEEDS): | |
| """The three claims a job has to earn: isolation, a resume that repaints nothing, and its cost.""" | |
| steps = tuple(Step("paint_roller", reads=3) for _ in range(1)) | |
| fresh, resumed, saved = [], [], [] | |
| for seed in seeds: | |
| state = Path(tmp_path) / f"job_{seed}.json" | |
| first = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state) | |
| second = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state) | |
| fresh.append(first) | |
| resumed.append(second) | |
| saved.append(1.0 - second["paint_l"] / first["paint_l"] if first["paint_l"] else 0.0) | |
| first, second = fresh[0], resumed[0] | |
| return { | |
| "walls": {wall_id: {"outcome": result["outcome"], "paint_l": result["paint_l"], | |
| "minutes": result["minutes"]} | |
| for wall_id, result in first["walls"].items()}, | |
| "outcome": first["outcome"], | |
| "walls_covered": first["walls_covered"], "walls_total": first["walls_total"], | |
| # A refused wall is a judgement, and a judgement must cost nothing and block nothing. | |
| "refused_wall_litres": first["walls"]["wall_2"]["paint_l"], | |
| "refused_wall_blocks_the_next": first["walls"]["wall_3"]["outcome"] == "not_started", | |
| "fresh_paint_l": first["paint_l"], "fresh_minutes": first["minutes"], | |
| "resume_paint_l": second["paint_l"], "resume_minutes": second["minutes"], | |
| "resume_repainted": second["walls"]["wall_1"]["outcome"] != "already_covered", | |
| "resume_saves": round(float(np.mean(saved)), 4), | |
| "resume_outcome": second["outcome"], | |
| "total_spend_paint_l": second["paint_l_total"], | |
| } | |
| def run_safety_probe(tmp_path, seeds=JOB_SEEDS): | |
| """A fault is not a judgement: it must stop the order, survive the process, and need a named reset.""" | |
| steps = (Step("paint_roller", reads=3),) | |
| halts = [] | |
| for seed in seeds: | |
| state = Path(tmp_path) / f"halt_{seed}.json" | |
| tripped = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state, stall_after=2, | |
| limits={"heartbeat_timeout_s": 1e-9}) | |
| after = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state) | |
| refused_blank = False | |
| try: | |
| reset_latch(state, " ") | |
| except SafeStop: | |
| refused_blank = True | |
| reset_latch(state, "benchmark operator") | |
| released = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state) | |
| halts.append({ | |
| "tripped_outcome": tripped["outcome"], | |
| "tripped_wall": tripped["walls"]["wall_1"]["outcome"], | |
| "walls_after_trip": {wall_id: result["outcome"] | |
| for wall_id, result in tripped["walls"].items()}, | |
| "litres_after_trip": tripped["paint_l"], | |
| "resumed_while_latched": after["outcome"], | |
| "resumed_litres_while_latched": after["paint_l"], | |
| "blank_reset_refused": refused_blank, | |
| "after_named_reset": released["outcome"], | |
| }) | |
| first = halts[0] | |
| first["all_seeds_agree"] = len({json.dumps(h, sort_keys=True) for h in halts}) == 1 | |
| return first | |
| def run_floor(seeds=SEEDS): | |
| """The open-loop floor: one pass, no re-observation, and it reports success regardless.""" | |
| runs = [] | |
| for scenario in ({"name": "floor_open_loop", "reads": 1, "budget": {}}, | |
| {"name": "floor_open_loop_misses", "reads": 1, "budget": {"miss_rate": 0.2}}): | |
| paintable, keepout = fixture() | |
| for seed in seeds: | |
| delivered = one_round_of_paint(paintable, keepout, scenario["budget"].get("miss_rate", 0.1), | |
| seed) | |
| target = cl.Budget().target | |
| runs.append({"name": scenario["name"], "outcome": "covered", # it always says this | |
| "coverage_true": round(delivered["coverage_true"], 4), | |
| "paint_l": delivered["paint_l"], "minutes": delivered["minutes"], | |
| "false_complete": delivered["coverage_true"] < target, "rounds": 1}) | |
| rows = [] | |
| for name in sorted({r["name"] for r in runs}): | |
| group = [r for r in runs if r["name"] == name] | |
| mean = lambda key: round(float(np.mean([r[key] for r in group])), 4) # noqa: E731 | |
| rows.append({"name": name, "outcomes": ["covered"], "runs": len(group), "rounds": 1.0, | |
| "coverage_true": mean("coverage_true"), "gap": mean("coverage_true"), | |
| "paint_l": mean("paint_l"), "minutes": mean("minutes"), | |
| "false_complete": sum(1 for r in group if r["false_complete"])}) | |
| return rows | |
| def evaluate_gates(scenarios, job, safety, silent, unnamed, rerun_identical): | |
| """Every gate is a number with a target. A gate that cannot be stated numerically is not a gate.""" | |
| by_name = {row["name"]: row for row in scenarios} | |
| runs = sum(row["runs"] for row in scenarios) | |
| return [ | |
| {"name": "no_silent_failure", "hard": True, "what": "a loop said finished on a wall that was not", | |
| "target": "0 runs", "measured": f"{silent} of {runs}", "passed": silent == 0}, | |
| {"name": "named_outcomes", "hard": True, "what": "every run ended in a named outcome, not a crash", | |
| "target": "0 runs outside the outcome list", "measured": f"{unnamed} of {runs}", | |
| "passed": unnamed == 0}, | |
| {"name": "gap_biased_3reads", "hard": False, "what": "|believed - true| coverage at 3 reads", | |
| "target": "<= 0.010", "measured": by_name["biased_3reads"]["gap"], | |
| "passed": by_name["biased_3reads"]["gap"] <= 0.010}, | |
| {"name": "gap_biased_1read", "hard": False, "what": "|believed - true| coverage at 1 read", | |
| "target": "<= 0.060", "measured": by_name["biased_1read"]["gap"], | |
| "passed": by_name["biased_1read"]["gap"] <= 0.060}, | |
| {"name": "ambiguity_gate_pays", "hard": False, | |
| "what": "litres when the reads disagree, gate on vs off", "target": "on <= off", | |
| "measured": f"{by_name['ambiguous_gate_on']['paint_l']} vs " | |
| f"{by_name['ambiguous_gate_off']['paint_l']}", | |
| "passed": by_name["ambiguous_gate_on"]["paint_l"] <= by_name["ambiguous_gate_off"]["paint_l"]}, | |
| {"name": "refusal_costs_nothing", "hard": True, "what": "litres on a wall the gate refused", | |
| "target": "== 0.000", "measured": job["refused_wall_litres"], | |
| "passed": job["refused_wall_litres"] == 0.0}, | |
| {"name": "failure_is_isolated", "hard": True, "what": "a refused wall stopped the next one", | |
| "target": "the order carries on", | |
| "measured": "yes" if job["refused_wall_blocks_the_next"] else "no", | |
| "passed": not job["refused_wall_blocks_the_next"]}, | |
| {"name": "resume_saves_paint", "hard": True, "what": "paint saved by resuming, nothing repainted", | |
| "target": ">= 0.40 and no repaint", | |
| "measured": f"{job['resume_saves']:.2f} saved, repainted={job['resume_repainted']}", | |
| "passed": job["resume_saves"] >= 0.40 and not job["resume_repainted"]}, | |
| {"name": "halt_is_sticky", "hard": True, | |
| "what": "a latched safe-stop halts the order and survives the process", | |
| "target": "safe_stop, then 0.000 L on resume", | |
| "measured": f"{safety['tripped_outcome']} -> {safety['resumed_while_latched']} " | |
| f"({safety['resumed_litres_while_latched']} L)", | |
| "passed": (safety["tripped_outcome"] == "safe_stop" | |
| and safety["resumed_while_latched"] == "safe_stop" | |
| and safety["resumed_litres_while_latched"] == 0.0)}, | |
| {"name": "halt_needs_a_name", "hard": True, "what": "a latch clears only for a named operator", | |
| "target": "blank refused, a named operator resumes", | |
| "measured": f"blank_refused={safety['blank_reset_refused']}, " | |
| f"after_reset={safety['after_named_reset']}", | |
| "passed": safety["blank_reset_refused"] and safety["after_named_reset"] != "safe_stop"}, | |
| {"name": "deterministic", "hard": True, "what": "the same seeds give the same numbers", | |
| "target": "identical on a second run of the suite", "measured": str(rerun_identical), | |
| "passed": rerun_identical}, | |
| ] | |
| def render_table(report): | |
| """The markdown a reader sees, in the same shape as the perception benchmark's table.""" | |
| lines = ["| Scenario | Outcome | Rounds | Paint L | Min | True | Believed | Gap | False complete |", | |
| "| --- | --- | --- | --- | --- | --- | --- | --- | --- |"] | |
| for row in report["scenarios"] + report["floor"]: | |
| believed = row.get("coverage_believed") | |
| lines.append(f"| `{row['name']}` | {'/'.join(row['outcomes'])} | {row['rounds']:.1f} | " | |
| f"{row['paint_l']:.3f} | {row['minutes']:.1f} | {row['coverage_true']:.3f} | " | |
| f"{believed:.3f} | {row['gap']:.4f} | {row['false_complete']} |" | |
| if believed is not None else | |
| f"| `{row['name']}` | {'/'.join(row['outcomes'])} | {row['rounds']:.1f} | " | |
| f"{row['paint_l']:.3f} | {row['minutes']:.1f} | {row['coverage_true']:.3f} | " | |
| f"n/a | {row['gap']:.4f} | {row['false_complete']} |") | |
| lines += ["", "| Gate | Target | Measured | Result |", "| --- | --- | --- | --- |"] | |
| for gate in report["gates"]: | |
| lines.append(f"| `{gate['name']}` | {gate['target']} | {gate['measured']} | " | |
| f"{'**pass**' if gate['passed'] else 'FAIL'} |") | |
| return "\n".join(lines) + "\n" | |
| def build_report(seeds=SEEDS, tmp_path=None): | |
| """Run the whole suite: loop scenarios, the job and safety probes, the floor, then the gates.""" | |
| import tempfile | |
| with tempfile.TemporaryDirectory() as tmp: | |
| tmp_path = Path(tmp_path or tmp) | |
| scenarios, silent, unnamed = run_loop_scenarios(seeds) | |
| again, _, _ = run_loop_scenarios(seeds) | |
| rerun_identical = json.dumps(scenarios, sort_keys=True) == json.dumps(again, sort_keys=True) | |
| job = run_job_probe(tmp_path, JOB_SEEDS) | |
| safety = run_safety_probe(tmp_path, JOB_SEEDS) | |
| floor = run_floor(seeds) | |
| gates = evaluate_gates(scenarios, job, safety, silent, unnamed, rerun_identical) | |
| report = {"suite": SUITE_VERSION, "scope": SCOPE, "fixture": "2.7 m wall plane, 10 mm/unit", | |
| "seeds": list(seeds), "engine": "numpy default_rng, seeded per scenario and round", | |
| "scenarios": scenarios, "job": job, "safety": safety, "floor": floor, "gates": gates, | |
| "hard_gates_passed": sum(1 for g in gates if g["hard"] and g["passed"]), | |
| "hard_gates": sum(1 for g in gates if g["hard"]), | |
| "passed": all(g["passed"] for g in gates if g["hard"])} | |
| report["failed_hard_gates"] = [g["name"] for g in gates if g["hard"] and not g["passed"]] | |
| return report | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__.splitlines()[0], | |
| formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--out", type=Path, help="write the full record here (JSON)") | |
| ap.add_argument("--table", type=Path, help="write the markdown table here") | |
| ap.add_argument("--registry", type=Path, help="append the run to a run_registry.py ledger") | |
| ap.add_argument("--seeds", type=int, default=len(SEEDS), help="how many of the fixed seeds to use") | |
| args = ap.parse_args() | |
| report = build_report(tuple(SEEDS[:args.seeds])) | |
| print(f"{'scenario':<28}{'outcome':<20}{'rounds':>7}{'paint L':>9}{'min':>7}" | |
| f"{'true':>7}{'believed':>9}{'gap':>8}") | |
| for row in report["scenarios"] + report["floor"]: | |
| believed = row.get("coverage_believed") | |
| shown = f"{believed:.3f}" if believed is not None else "n/a" | |
| print(f"{row['name']:<28}{'/'.join(row['outcomes']):<20}{row['rounds']:>7.1f}" | |
| f"{row['paint_l']:>9.3f}{row['minutes']:>7.1f}{row['coverage_true']:>7.3f}" | |
| f"{shown:>9}{row['gap']:>8.4f}") | |
| print() | |
| for gate in report["gates"]: | |
| print(f" [{'pass' if gate['passed'] else 'FAIL'}] {gate['name']:<22}{gate['target']:<36}" | |
| f"{gate['measured']}") | |
| print(f"\nhard gates: {report['hard_gates_passed']}/{report['hard_gates']} " | |
| f"-> {'PASS' if report['passed'] else 'FAIL'}") | |
| if args.out: | |
| args.out.parent.mkdir(parents=True, exist_ok=True) | |
| args.out.write_text(json.dumps(report, indent=2) + "\n") | |
| print(f"record: {args.out}") | |
| if args.table: | |
| args.table.parent.mkdir(parents=True, exist_ok=True) | |
| args.table.write_text(render_table(report)) | |
| print(f"table: {args.table}") | |
| if args.registry: | |
| from run_registry import append | |
| record = append(args.registry, "eval", arch=f"robotics/{SUITE_VERSION}", | |
| config={"seeds": list(report["seeds"]), "scenarios": len(report["scenarios"])}, | |
| metrics={"hard_gates_passed": report["hard_gates_passed"], | |
| "hard_gates": report["hard_gates"]}, | |
| extra={"suite": SUITE_VERSION, "scope": SCOPE}) | |
| print(f"ledger: {args.registry} seq {record['seq']} ({record['hash'][:12]})") | |
| raise SystemExit(0 if report["passed"] else 1) | |
| if __name__ == "__main__": | |
| main() | |