#!/usr/bin/env python3 """A fixed protocol for the robotics layer: same scenarios, same seeds, every run. `benchmark/SPEC.md` does this for perception -- one protocol, one pinned dataset, one emitted record a stranger can re-derive. The robotics half (`coverage_loop.py`, `mission.py`, `safety_supervisor.py`) has had no equivalent: its numbers lived in prose, hand-run, in a doc. That is what the roadmap's standing requirements forbid (*an unlogged result cannot be cited*). This is the missing protocol. **What is measured.** Every scenario is the same 2.7 m wall plane on the same three seeds, and each one isolates a single stressor -- a refused room, an unreadable patch, a biased or systematically wrong re-scan, undelivered passes, a controller that stops proving it is alive. The metrics are the ones that can be wrong in a way that matters: * ``false_complete`` -- the loop said finished and the wall was *not*. The only unforgivable metric. * ``gap`` -- |believed - true| coverage. How far the machine's picture is from the wall. * ``paint_l`` / ``minutes`` / ``rounds`` -- what the verdict cost. * ``outcome`` -- must be one of the **named** outcomes, every time. A crash or a hang is a failure. * job-level: litres spent on a wall the gate refused (**must be 0**), what a resume saves, and whether a latched safe-stop stops the *order* and survives the process. **The floor.** An open-loop controller -- plan once, paint once, declare the wall done, never re-observe -- is reported alongside. It is cheap and it always reports success, so it can only be judged on ``false_complete``; it fails that gate, which is the whole justification for a closed loop existing. That mirrors ``floor:all-other`` in the perception benchmark, and it is what the retired ``hardware.closed_loop`` demo produced. **Honest scope.** The observation is *simulated* with the estimator error measured elsewhere (`docs/HARDWARE.md`), not taken from a camera on a machine. This benchmark gates the control and safety logic, not the perception it consumes, and the emitted record says so in ``scope``. python3 benchmark_robotics.py --out benchmark/records/robotics_1.json """ from __future__ import annotations import argparse import json from pathlib import Path import numpy as np import coverage_loop as cl import hardware as hw from mission import JobWall, Step, reset_latch, run_job from robot_planner import build_plan from safety_supervisor import SafeStop SUITE_VERSION = "robotics/1.0" MM_PER_UNIT = 10.0 SEEDS = (0, 1, 2) # every loop scenario runs on all three; job scenarios on the first JOB_SEEDS = (0,) PLATFORM = "mast_arm" APPROVED = {"decision": "approve"} REJECTED = {"decision": "reject", "reasons": ["no operator approval for this room"]} SCOPE = ("software: the observation is simulated with the measured estimator error; no camera, no machine") # One stressor per scenario, so a red gate names the thing that broke. `budget` is merged into Budget, # `reads` overrides it, and `stall_after` is a controller that goes quiet (with a finite watchdog). SCENARIOS = ( {"name": "clean_3reads", "reads": 3, "budget": {}}, {"name": "clean_1read", "reads": 1, "budget": {}}, {"name": "biased_1read", "reads": 1, "budget": {"under_claim": 0.05}}, {"name": "biased_3reads", "reads": 3, "budget": {"under_claim": 0.05}}, {"name": "systematic_bias_3reads", "reads": 3, "budget": {"under_claim": 0.05, "systematic": True}}, {"name": "overclaim_3reads", "reads": 3, "budget": {"over_claim": 0.3}}, {"name": "gate_refused", "reads": 3, "budget": {}, "decision": REJECTED}, {"name": "ambiguous_gate_on", "reads": 3, "budget": {"ambiguous_patch": 0.15}}, {"name": "ambiguous_gate_off", "reads": 3, "budget": {"ambiguous_patch": 0.15, "max_ambiguous": 1.0}}, {"name": "undelivered_passes", "reads": 3, "budget": {"miss_rate": 0.5}}, {"name": "target_above_geometry", "reads": 3, "budget": {"target": 0.999}}, {"name": "nothing_to_paint", "reads": 3, "budget": {}, "empty": True}, {"name": "stale_controller", "reads": 3, "budget": {}, "stall_after": 2}, ) NAMED_OUTCOMES = ("covered", "stalled", "budget_exhausted", "time_exhausted", "rounds_exhausted", "nothing_left_to_plan", "ambiguous", "refused", "safety_stop", "nothing_to_paint") def fixture(empty=False): paintable, keepout = cl.wall() if empty: paintable = np.zeros_like(paintable) return paintable, keepout def supervisor_for(paintable, limits): return hw.supervisor_for(paintable.shape[1] * MM_PER_UNIT, paintable.shape[0] * MM_PER_UNIT, PLATFORM, limits=limits) def one_round_of_paint(paintable, keepout, miss_rate, seed, operation="paint_roller", mm_per_unit=MM_PER_UNIT): """What a single open-loop pass delivers, with no re-observation at all. The floor's engine.""" tool, brush = hw.TOOLS[hw.OPERATIONS[operation]], hw.TOOLS["brush"] target = np.asarray(paintable, bool) & ~np.asarray(keepout, bool) height_mm = paintable.shape[0] * mm_per_unit plan, _ = build_plan(paintable, keepout, stroke_width=max(2, int(tool.width_mm / mm_per_unit)), overlap=tool.overlap, margin=0, orientation="horizontal", corners=None, mm_per_unit=mm_per_unit, boundary_tolerance=2.0, already_painted=np.zeros_like(target), trim_width=max(2, int(brush.width_mm / mm_per_unit)), trim_overlap=brush.overlap) strokes, trim = hw.strokes_from_plan(plan)[0], hw.strokes_from_plan(plan, trim=True)[0] delivered, minutes = np.zeros_like(target), 0.0 rng = np.random.default_rng(seed) for stroke_set, stroke_op in ((strokes, operation), (trim, "cut_in")): if not stroke_set: continue job = hw.compile_job(stroke_set, height_mm, stroke_op, PLATFORM, wall_id="floor") missed = set(np.flatnonzero(rng.random(len(job["commands"])) < miss_rate)) delivered |= hw.rasterize(job, target.shape, mm_per_unit, height_mm, miss=lambda i, missed=missed: i in missed) & target minutes += job["totals"]["duration_min"] volume = cl.litres(delivered, mm_per_unit, cl.Budget()) return {"coverage_true": float((delivered & target).sum() / target.sum()), "paint_l": round(volume, 3), "minutes": round(minutes, 1)} def run_scenario(scenario, seed): """One scenario, one seed: the loop's verdict plus what it cost and what the verdict was worth.""" paintable, keepout = fixture(scenario.get("empty", False)) budget = cl.Budget(reads=scenario.get("reads", 3), **scenario.get("budget", {})) supervisor = None if scenario.get("stall_after") is not None: supervisor = supervisor_for(paintable, {"heartbeat_timeout_s": 1e-9}) result = cl.run(paintable, keepout, MM_PER_UNIT, scenario.get("decision", APPROVED), budget, operation=scenario.get("operation", "paint_roller"), platform=PLATFORM, wall_id=scenario["name"], seed=seed, supervisor=supervisor, stall_after=scenario.get("stall_after")) return {"outcome": result["outcome"], "rounds": len(result["rounds"]), "coverage_true": result["coverage_true"], "coverage_believed": result["coverage_observed"], "gap": round(abs(result["coverage_observed"] - result["coverage_true"]), 4), "false_complete": result["false_complete"], "latched": result["latched"], "paint_l": result["paint_l"], "minutes": result["minutes"], "reason": result["reason"]} def run_loop_scenarios(seeds=SEEDS): """Every scenario on every seed, reduced to a mean and an outcome set (unstable outcomes are named).""" rows, silent, unnamed = [], 0, 0 for scenario in SCENARIOS: runs = [run_scenario(scenario, seed) for seed in seeds] outcomes = sorted({r["outcome"] for r in runs}) silent += sum(1 for r in runs if r["false_complete"]) unnamed += sum(1 for r in runs if r["outcome"] not in NAMED_OUTCOMES) mean = lambda key: round(float(np.mean([r[key] for r in runs])), 4) # noqa: E731 rows.append({"name": scenario["name"], "outcomes": outcomes, "outcome_stable": len(outcomes) == 1, "runs": len(runs), "gap": mean("gap"), "rounds": mean("rounds"), "paint_l": mean("paint_l"), "minutes": mean("minutes"), "coverage_true": mean("coverage_true"), "coverage_believed": mean("coverage_believed"), "false_complete": sum(1 for r in runs if r["false_complete"]), "reason": runs[0]["reason"]}) return rows, silent, unnamed def job_walls(steps, unreadable=True): """The order the layer is built for: one clean wall, one the gate refuses, one it cannot read.""" paintable, keepout = fixture() return [JobWall("wall_1", paintable.copy(), keepout.copy(), steps, dict(APPROVED), {"under_claim": 0.05}), JobWall("wall_2", paintable.copy(), keepout.copy(), steps, dict(REJECTED), {}), JobWall("wall_3", paintable.copy(), keepout.copy(), steps, dict(APPROVED), {"ambiguous_patch": 0.15} if unreadable else {"under_claim": 0.05})] def run_job_probe(tmp_path, seeds=JOB_SEEDS): """The three claims a job has to earn: isolation, a resume that repaints nothing, and its cost.""" steps = tuple(Step("paint_roller", reads=3) for _ in range(1)) fresh, resumed, saved = [], [], [] for seed in seeds: state = Path(tmp_path) / f"job_{seed}.json" first = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state) second = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state) fresh.append(first) resumed.append(second) saved.append(1.0 - second["paint_l"] / first["paint_l"] if first["paint_l"] else 0.0) first, second = fresh[0], resumed[0] return { "walls": {wall_id: {"outcome": result["outcome"], "paint_l": result["paint_l"], "minutes": result["minutes"]} for wall_id, result in first["walls"].items()}, "outcome": first["outcome"], "walls_covered": first["walls_covered"], "walls_total": first["walls_total"], # A refused wall is a judgement, and a judgement must cost nothing and block nothing. "refused_wall_litres": first["walls"]["wall_2"]["paint_l"], "refused_wall_blocks_the_next": first["walls"]["wall_3"]["outcome"] == "not_started", "fresh_paint_l": first["paint_l"], "fresh_minutes": first["minutes"], "resume_paint_l": second["paint_l"], "resume_minutes": second["minutes"], "resume_repainted": second["walls"]["wall_1"]["outcome"] != "already_covered", "resume_saves": round(float(np.mean(saved)), 4), "resume_outcome": second["outcome"], "total_spend_paint_l": second["paint_l_total"], } def run_safety_probe(tmp_path, seeds=JOB_SEEDS): """A fault is not a judgement: it must stop the order, survive the process, and need a named reset.""" steps = (Step("paint_roller", reads=3),) halts = [] for seed in seeds: state = Path(tmp_path) / f"halt_{seed}.json" tripped = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state, stall_after=2, limits={"heartbeat_timeout_s": 1e-9}) after = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state) refused_blank = False try: reset_latch(state, " ") except SafeStop: refused_blank = True reset_latch(state, "benchmark operator") released = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state) halts.append({ "tripped_outcome": tripped["outcome"], "tripped_wall": tripped["walls"]["wall_1"]["outcome"], "walls_after_trip": {wall_id: result["outcome"] for wall_id, result in tripped["walls"].items()}, "litres_after_trip": tripped["paint_l"], "resumed_while_latched": after["outcome"], "resumed_litres_while_latched": after["paint_l"], "blank_reset_refused": refused_blank, "after_named_reset": released["outcome"], }) first = halts[0] first["all_seeds_agree"] = len({json.dumps(h, sort_keys=True) for h in halts}) == 1 return first def run_floor(seeds=SEEDS): """The open-loop floor: one pass, no re-observation, and it reports success regardless.""" runs = [] for scenario in ({"name": "floor_open_loop", "reads": 1, "budget": {}}, {"name": "floor_open_loop_misses", "reads": 1, "budget": {"miss_rate": 0.2}}): paintable, keepout = fixture() for seed in seeds: delivered = one_round_of_paint(paintable, keepout, scenario["budget"].get("miss_rate", 0.1), seed) target = cl.Budget().target runs.append({"name": scenario["name"], "outcome": "covered", # it always says this "coverage_true": round(delivered["coverage_true"], 4), "paint_l": delivered["paint_l"], "minutes": delivered["minutes"], "false_complete": delivered["coverage_true"] < target, "rounds": 1}) rows = [] for name in sorted({r["name"] for r in runs}): group = [r for r in runs if r["name"] == name] mean = lambda key: round(float(np.mean([r[key] for r in group])), 4) # noqa: E731 rows.append({"name": name, "outcomes": ["covered"], "runs": len(group), "rounds": 1.0, "coverage_true": mean("coverage_true"), "gap": mean("coverage_true"), "paint_l": mean("paint_l"), "minutes": mean("minutes"), "false_complete": sum(1 for r in group if r["false_complete"])}) return rows def evaluate_gates(scenarios, job, safety, silent, unnamed, rerun_identical): """Every gate is a number with a target. A gate that cannot be stated numerically is not a gate.""" by_name = {row["name"]: row for row in scenarios} runs = sum(row["runs"] for row in scenarios) return [ {"name": "no_silent_failure", "hard": True, "what": "a loop said finished on a wall that was not", "target": "0 runs", "measured": f"{silent} of {runs}", "passed": silent == 0}, {"name": "named_outcomes", "hard": True, "what": "every run ended in a named outcome, not a crash", "target": "0 runs outside the outcome list", "measured": f"{unnamed} of {runs}", "passed": unnamed == 0}, {"name": "gap_biased_3reads", "hard": False, "what": "|believed - true| coverage at 3 reads", "target": "<= 0.010", "measured": by_name["biased_3reads"]["gap"], "passed": by_name["biased_3reads"]["gap"] <= 0.010}, {"name": "gap_biased_1read", "hard": False, "what": "|believed - true| coverage at 1 read", "target": "<= 0.060", "measured": by_name["biased_1read"]["gap"], "passed": by_name["biased_1read"]["gap"] <= 0.060}, {"name": "ambiguity_gate_pays", "hard": False, "what": "litres when the reads disagree, gate on vs off", "target": "on <= off", "measured": f"{by_name['ambiguous_gate_on']['paint_l']} vs " f"{by_name['ambiguous_gate_off']['paint_l']}", "passed": by_name["ambiguous_gate_on"]["paint_l"] <= by_name["ambiguous_gate_off"]["paint_l"]}, {"name": "refusal_costs_nothing", "hard": True, "what": "litres on a wall the gate refused", "target": "== 0.000", "measured": job["refused_wall_litres"], "passed": job["refused_wall_litres"] == 0.0}, {"name": "failure_is_isolated", "hard": True, "what": "a refused wall stopped the next one", "target": "the order carries on", "measured": "yes" if job["refused_wall_blocks_the_next"] else "no", "passed": not job["refused_wall_blocks_the_next"]}, {"name": "resume_saves_paint", "hard": True, "what": "paint saved by resuming, nothing repainted", "target": ">= 0.40 and no repaint", "measured": f"{job['resume_saves']:.2f} saved, repainted={job['resume_repainted']}", "passed": job["resume_saves"] >= 0.40 and not job["resume_repainted"]}, {"name": "halt_is_sticky", "hard": True, "what": "a latched safe-stop halts the order and survives the process", "target": "safe_stop, then 0.000 L on resume", "measured": f"{safety['tripped_outcome']} -> {safety['resumed_while_latched']} " f"({safety['resumed_litres_while_latched']} L)", "passed": (safety["tripped_outcome"] == "safe_stop" and safety["resumed_while_latched"] == "safe_stop" and safety["resumed_litres_while_latched"] == 0.0)}, {"name": "halt_needs_a_name", "hard": True, "what": "a latch clears only for a named operator", "target": "blank refused, a named operator resumes", "measured": f"blank_refused={safety['blank_reset_refused']}, " f"after_reset={safety['after_named_reset']}", "passed": safety["blank_reset_refused"] and safety["after_named_reset"] != "safe_stop"}, {"name": "deterministic", "hard": True, "what": "the same seeds give the same numbers", "target": "identical on a second run of the suite", "measured": str(rerun_identical), "passed": rerun_identical}, ] def render_table(report): """The markdown a reader sees, in the same shape as the perception benchmark's table.""" lines = ["| Scenario | Outcome | Rounds | Paint L | Min | True | Believed | Gap | False complete |", "| --- | --- | --- | --- | --- | --- | --- | --- | --- |"] for row in report["scenarios"] + report["floor"]: believed = row.get("coverage_believed") lines.append(f"| `{row['name']}` | {'/'.join(row['outcomes'])} | {row['rounds']:.1f} | " f"{row['paint_l']:.3f} | {row['minutes']:.1f} | {row['coverage_true']:.3f} | " f"{believed:.3f} | {row['gap']:.4f} | {row['false_complete']} |" if believed is not None else f"| `{row['name']}` | {'/'.join(row['outcomes'])} | {row['rounds']:.1f} | " f"{row['paint_l']:.3f} | {row['minutes']:.1f} | {row['coverage_true']:.3f} | " f"n/a | {row['gap']:.4f} | {row['false_complete']} |") lines += ["", "| Gate | Target | Measured | Result |", "| --- | --- | --- | --- |"] for gate in report["gates"]: lines.append(f"| `{gate['name']}` | {gate['target']} | {gate['measured']} | " f"{'**pass**' if gate['passed'] else 'FAIL'} |") return "\n".join(lines) + "\n" def build_report(seeds=SEEDS, tmp_path=None): """Run the whole suite: loop scenarios, the job and safety probes, the floor, then the gates.""" import tempfile with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp_path or tmp) scenarios, silent, unnamed = run_loop_scenarios(seeds) again, _, _ = run_loop_scenarios(seeds) rerun_identical = json.dumps(scenarios, sort_keys=True) == json.dumps(again, sort_keys=True) job = run_job_probe(tmp_path, JOB_SEEDS) safety = run_safety_probe(tmp_path, JOB_SEEDS) floor = run_floor(seeds) gates = evaluate_gates(scenarios, job, safety, silent, unnamed, rerun_identical) report = {"suite": SUITE_VERSION, "scope": SCOPE, "fixture": "2.7 m wall plane, 10 mm/unit", "seeds": list(seeds), "engine": "numpy default_rng, seeded per scenario and round", "scenarios": scenarios, "job": job, "safety": safety, "floor": floor, "gates": gates, "hard_gates_passed": sum(1 for g in gates if g["hard"] and g["passed"]), "hard_gates": sum(1 for g in gates if g["hard"]), "passed": all(g["passed"] for g in gates if g["hard"])} report["failed_hard_gates"] = [g["name"] for g in gates if g["hard"] and not g["passed"]] return report def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0], formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--out", type=Path, help="write the full record here (JSON)") ap.add_argument("--table", type=Path, help="write the markdown table here") ap.add_argument("--registry", type=Path, help="append the run to a run_registry.py ledger") ap.add_argument("--seeds", type=int, default=len(SEEDS), help="how many of the fixed seeds to use") args = ap.parse_args() report = build_report(tuple(SEEDS[:args.seeds])) print(f"{'scenario':<28}{'outcome':<20}{'rounds':>7}{'paint L':>9}{'min':>7}" f"{'true':>7}{'believed':>9}{'gap':>8}") for row in report["scenarios"] + report["floor"]: believed = row.get("coverage_believed") shown = f"{believed:.3f}" if believed is not None else "n/a" print(f"{row['name']:<28}{'/'.join(row['outcomes']):<20}{row['rounds']:>7.1f}" f"{row['paint_l']:>9.3f}{row['minutes']:>7.1f}{row['coverage_true']:>7.3f}" f"{shown:>9}{row['gap']:>8.4f}") print() for gate in report["gates"]: print(f" [{'pass' if gate['passed'] else 'FAIL'}] {gate['name']:<22}{gate['target']:<36}" f"{gate['measured']}") print(f"\nhard gates: {report['hard_gates_passed']}/{report['hard_gates']} " f"-> {'PASS' if report['passed'] else 'FAIL'}") if args.out: args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(json.dumps(report, indent=2) + "\n") print(f"record: {args.out}") if args.table: args.table.parent.mkdir(parents=True, exist_ok=True) args.table.write_text(render_table(report)) print(f"table: {args.table}") if args.registry: from run_registry import append record = append(args.registry, "eval", arch=f"robotics/{SUITE_VERSION}", config={"seeds": list(report["seeds"]), "scenarios": len(report["scenarios"])}, metrics={"hard_gates_passed": report["hard_gates_passed"], "hard_gates": report["hard_gates"]}, extra={"suite": SUITE_VERSION, "scope": SCOPE}) print(f"ledger: {args.registry} seq {record['seq']} ({record['hash'][:12]})") raise SystemExit(0 if report["passed"] else 1) if __name__ == "__main__": main()