painting-vision-robotics-kit / benchmark_robotics.py
constructelligence's picture
Update code: real-machine G-code adapter, commissioning gate, machine_job runner, pump watchdog firmware, twin completion gate, realism and masking modules (card and weights unchanged)
4eecd03 verified
Raw History Blame Contribute Delete
22.9 kB
#!/usr/bin/env python3
"""A fixed protocol for the robotics layer: same scenarios, same seeds, every run.
`benchmark/SPEC.md` does this for perception -- one protocol, one pinned dataset, one emitted record a
stranger can re-derive. The robotics half (`coverage_loop.py`, `mission.py`, `safety_supervisor.py`) has
had no equivalent: its numbers lived in prose, hand-run, in a doc. That is what the roadmap's standing
requirements forbid (*an unlogged result cannot be cited*). This is the missing protocol.
**What is measured.** Every scenario is the same 2.7 m wall plane on the same three seeds, and each one
isolates a single stressor -- a refused room, an unreadable patch, a biased or systematically wrong
re-scan, undelivered passes, a controller that stops proving it is alive. The metrics are the ones that
can be wrong in a way that matters:
* ``false_complete`` -- the loop said finished and the wall was *not*. The only unforgivable metric.
* ``gap`` -- |believed - true| coverage. How far the machine's picture is from the wall.
* ``paint_l`` / ``minutes`` / ``rounds`` -- what the verdict cost.
* ``outcome`` -- must be one of the **named** outcomes, every time. A crash or a hang is a failure.
* job-level: litres spent on a wall the gate refused (**must be 0**), what a resume saves, and whether a
latched safe-stop stops the *order* and survives the process.
**The floor.** An open-loop controller -- plan once, paint once, declare the wall done, never re-observe
-- is reported alongside. It is cheap and it always reports success, so it can only be judged on
``false_complete``; it fails that gate, which is the whole justification for a closed loop existing.
That mirrors ``floor:all-other`` in the perception benchmark, and it is what the retired
``hardware.closed_loop`` demo produced.
**Honest scope.** The observation is *simulated* with the estimator error measured elsewhere
(`docs/HARDWARE.md`), not taken from a camera on a machine. This benchmark gates the control and safety
logic, not the perception it consumes, and the emitted record says so in ``scope``.
python3 benchmark_robotics.py --out benchmark/records/robotics_1.json
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import numpy as np
import coverage_loop as cl
import hardware as hw
from mission import JobWall, Step, reset_latch, run_job
from robot_planner import build_plan
from safety_supervisor import SafeStop
SUITE_VERSION = "robotics/1.0"
MM_PER_UNIT = 10.0
SEEDS = (0, 1, 2) # every loop scenario runs on all three; job scenarios on the first
JOB_SEEDS = (0,)
PLATFORM = "mast_arm"
APPROVED = {"decision": "approve"}
REJECTED = {"decision": "reject", "reasons": ["no operator approval for this room"]}
SCOPE = ("software: the observation is simulated with the measured estimator error; no camera, no machine")
# One stressor per scenario, so a red gate names the thing that broke. `budget` is merged into Budget,
# `reads` overrides it, and `stall_after` is a controller that goes quiet (with a finite watchdog).
SCENARIOS = (
{"name": "clean_3reads", "reads": 3, "budget": {}},
{"name": "clean_1read", "reads": 1, "budget": {}},
{"name": "biased_1read", "reads": 1, "budget": {"under_claim": 0.05}},
{"name": "biased_3reads", "reads": 3, "budget": {"under_claim": 0.05}},
{"name": "systematic_bias_3reads", "reads": 3, "budget": {"under_claim": 0.05, "systematic": True}},
{"name": "overclaim_3reads", "reads": 3, "budget": {"over_claim": 0.3}},
{"name": "gate_refused", "reads": 3, "budget": {}, "decision": REJECTED},
{"name": "ambiguous_gate_on", "reads": 3, "budget": {"ambiguous_patch": 0.15}},
{"name": "ambiguous_gate_off", "reads": 3, "budget": {"ambiguous_patch": 0.15, "max_ambiguous": 1.0}},
{"name": "undelivered_passes", "reads": 3, "budget": {"miss_rate": 0.5}},
{"name": "target_above_geometry", "reads": 3, "budget": {"target": 0.999}},
{"name": "nothing_to_paint", "reads": 3, "budget": {}, "empty": True},
{"name": "stale_controller", "reads": 3, "budget": {}, "stall_after": 2},
)
NAMED_OUTCOMES = ("covered", "stalled", "budget_exhausted", "time_exhausted", "rounds_exhausted",
"nothing_left_to_plan", "ambiguous", "refused", "safety_stop", "nothing_to_paint")
def fixture(empty=False):
paintable, keepout = cl.wall()
if empty:
paintable = np.zeros_like(paintable)
return paintable, keepout
def supervisor_for(paintable, limits):
return hw.supervisor_for(paintable.shape[1] * MM_PER_UNIT, paintable.shape[0] * MM_PER_UNIT,
PLATFORM, limits=limits)
def one_round_of_paint(paintable, keepout, miss_rate, seed, operation="paint_roller",
mm_per_unit=MM_PER_UNIT):
"""What a single open-loop pass delivers, with no re-observation at all. The floor's engine."""
tool, brush = hw.TOOLS[hw.OPERATIONS[operation]], hw.TOOLS["brush"]
target = np.asarray(paintable, bool) & ~np.asarray(keepout, bool)
height_mm = paintable.shape[0] * mm_per_unit
plan, _ = build_plan(paintable, keepout, stroke_width=max(2, int(tool.width_mm / mm_per_unit)),
overlap=tool.overlap, margin=0, orientation="horizontal", corners=None,
mm_per_unit=mm_per_unit, boundary_tolerance=2.0,
already_painted=np.zeros_like(target),
trim_width=max(2, int(brush.width_mm / mm_per_unit)),
trim_overlap=brush.overlap)
strokes, trim = hw.strokes_from_plan(plan)[0], hw.strokes_from_plan(plan, trim=True)[0]
delivered, minutes = np.zeros_like(target), 0.0
rng = np.random.default_rng(seed)
for stroke_set, stroke_op in ((strokes, operation), (trim, "cut_in")):
if not stroke_set:
continue
job = hw.compile_job(stroke_set, height_mm, stroke_op, PLATFORM, wall_id="floor")
missed = set(np.flatnonzero(rng.random(len(job["commands"])) < miss_rate))
delivered |= hw.rasterize(job, target.shape, mm_per_unit, height_mm,
miss=lambda i, missed=missed: i in missed) & target
minutes += job["totals"]["duration_min"]
volume = cl.litres(delivered, mm_per_unit, cl.Budget())
return {"coverage_true": float((delivered & target).sum() / target.sum()),
"paint_l": round(volume, 3), "minutes": round(minutes, 1)}
def run_scenario(scenario, seed):
"""One scenario, one seed: the loop's verdict plus what it cost and what the verdict was worth."""
paintable, keepout = fixture(scenario.get("empty", False))
budget = cl.Budget(reads=scenario.get("reads", 3), **scenario.get("budget", {}))
supervisor = None
if scenario.get("stall_after") is not None:
supervisor = supervisor_for(paintable, {"heartbeat_timeout_s": 1e-9})
result = cl.run(paintable, keepout, MM_PER_UNIT, scenario.get("decision", APPROVED), budget,
operation=scenario.get("operation", "paint_roller"), platform=PLATFORM,
wall_id=scenario["name"], seed=seed, supervisor=supervisor,
stall_after=scenario.get("stall_after"))
return {"outcome": result["outcome"], "rounds": len(result["rounds"]),
"coverage_true": result["coverage_true"], "coverage_believed": result["coverage_observed"],
"gap": round(abs(result["coverage_observed"] - result["coverage_true"]), 4),
"false_complete": result["false_complete"], "latched": result["latched"],
"paint_l": result["paint_l"], "minutes": result["minutes"],
"reason": result["reason"]}
def run_loop_scenarios(seeds=SEEDS):
"""Every scenario on every seed, reduced to a mean and an outcome set (unstable outcomes are named)."""
rows, silent, unnamed = [], 0, 0
for scenario in SCENARIOS:
runs = [run_scenario(scenario, seed) for seed in seeds]
outcomes = sorted({r["outcome"] for r in runs})
silent += sum(1 for r in runs if r["false_complete"])
unnamed += sum(1 for r in runs if r["outcome"] not in NAMED_OUTCOMES)
mean = lambda key: round(float(np.mean([r[key] for r in runs])), 4) # noqa: E731
rows.append({"name": scenario["name"], "outcomes": outcomes,
"outcome_stable": len(outcomes) == 1, "runs": len(runs),
"gap": mean("gap"), "rounds": mean("rounds"), "paint_l": mean("paint_l"),
"minutes": mean("minutes"), "coverage_true": mean("coverage_true"),
"coverage_believed": mean("coverage_believed"),
"false_complete": sum(1 for r in runs if r["false_complete"]),
"reason": runs[0]["reason"]})
return rows, silent, unnamed
def job_walls(steps, unreadable=True):
"""The order the layer is built for: one clean wall, one the gate refuses, one it cannot read."""
paintable, keepout = fixture()
return [JobWall("wall_1", paintable.copy(), keepout.copy(), steps, dict(APPROVED),
{"under_claim": 0.05}),
JobWall("wall_2", paintable.copy(), keepout.copy(), steps, dict(REJECTED), {}),
JobWall("wall_3", paintable.copy(), keepout.copy(), steps, dict(APPROVED),
{"ambiguous_patch": 0.15} if unreadable else {"under_claim": 0.05})]
def run_job_probe(tmp_path, seeds=JOB_SEEDS):
"""The three claims a job has to earn: isolation, a resume that repaints nothing, and its cost."""
steps = tuple(Step("paint_roller", reads=3) for _ in range(1))
fresh, resumed, saved = [], [], []
for seed in seeds:
state = Path(tmp_path) / f"job_{seed}.json"
first = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state)
second = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state)
fresh.append(first)
resumed.append(second)
saved.append(1.0 - second["paint_l"] / first["paint_l"] if first["paint_l"] else 0.0)
first, second = fresh[0], resumed[0]
return {
"walls": {wall_id: {"outcome": result["outcome"], "paint_l": result["paint_l"],
"minutes": result["minutes"]}
for wall_id, result in first["walls"].items()},
"outcome": first["outcome"],
"walls_covered": first["walls_covered"], "walls_total": first["walls_total"],
# A refused wall is a judgement, and a judgement must cost nothing and block nothing.
"refused_wall_litres": first["walls"]["wall_2"]["paint_l"],
"refused_wall_blocks_the_next": first["walls"]["wall_3"]["outcome"] == "not_started",
"fresh_paint_l": first["paint_l"], "fresh_minutes": first["minutes"],
"resume_paint_l": second["paint_l"], "resume_minutes": second["minutes"],
"resume_repainted": second["walls"]["wall_1"]["outcome"] != "already_covered",
"resume_saves": round(float(np.mean(saved)), 4),
"resume_outcome": second["outcome"],
"total_spend_paint_l": second["paint_l_total"],
}
def run_safety_probe(tmp_path, seeds=JOB_SEEDS):
"""A fault is not a judgement: it must stop the order, survive the process, and need a named reset."""
steps = (Step("paint_roller", reads=3),)
halts = []
for seed in seeds:
state = Path(tmp_path) / f"halt_{seed}.json"
tripped = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state, stall_after=2,
limits={"heartbeat_timeout_s": 1e-9})
after = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state)
refused_blank = False
try:
reset_latch(state, " ")
except SafeStop:
refused_blank = True
reset_latch(state, "benchmark operator")
released = run_job(job_walls(steps), MM_PER_UNIT, seed=seed, state_path=state)
halts.append({
"tripped_outcome": tripped["outcome"],
"tripped_wall": tripped["walls"]["wall_1"]["outcome"],
"walls_after_trip": {wall_id: result["outcome"]
for wall_id, result in tripped["walls"].items()},
"litres_after_trip": tripped["paint_l"],
"resumed_while_latched": after["outcome"],
"resumed_litres_while_latched": after["paint_l"],
"blank_reset_refused": refused_blank,
"after_named_reset": released["outcome"],
})
first = halts[0]
first["all_seeds_agree"] = len({json.dumps(h, sort_keys=True) for h in halts}) == 1
return first
def run_floor(seeds=SEEDS):
"""The open-loop floor: one pass, no re-observation, and it reports success regardless."""
runs = []
for scenario in ({"name": "floor_open_loop", "reads": 1, "budget": {}},
{"name": "floor_open_loop_misses", "reads": 1, "budget": {"miss_rate": 0.2}}):
paintable, keepout = fixture()
for seed in seeds:
delivered = one_round_of_paint(paintable, keepout, scenario["budget"].get("miss_rate", 0.1),
seed)
target = cl.Budget().target
runs.append({"name": scenario["name"], "outcome": "covered", # it always says this
"coverage_true": round(delivered["coverage_true"], 4),
"paint_l": delivered["paint_l"], "minutes": delivered["minutes"],
"false_complete": delivered["coverage_true"] < target, "rounds": 1})
rows = []
for name in sorted({r["name"] for r in runs}):
group = [r for r in runs if r["name"] == name]
mean = lambda key: round(float(np.mean([r[key] for r in group])), 4) # noqa: E731
rows.append({"name": name, "outcomes": ["covered"], "runs": len(group), "rounds": 1.0,
"coverage_true": mean("coverage_true"), "gap": mean("coverage_true"),
"paint_l": mean("paint_l"), "minutes": mean("minutes"),
"false_complete": sum(1 for r in group if r["false_complete"])})
return rows
def evaluate_gates(scenarios, job, safety, silent, unnamed, rerun_identical):
"""Every gate is a number with a target. A gate that cannot be stated numerically is not a gate."""
by_name = {row["name"]: row for row in scenarios}
runs = sum(row["runs"] for row in scenarios)
return [
{"name": "no_silent_failure", "hard": True, "what": "a loop said finished on a wall that was not",
"target": "0 runs", "measured": f"{silent} of {runs}", "passed": silent == 0},
{"name": "named_outcomes", "hard": True, "what": "every run ended in a named outcome, not a crash",
"target": "0 runs outside the outcome list", "measured": f"{unnamed} of {runs}",
"passed": unnamed == 0},
{"name": "gap_biased_3reads", "hard": False, "what": "|believed - true| coverage at 3 reads",
"target": "<= 0.010", "measured": by_name["biased_3reads"]["gap"],
"passed": by_name["biased_3reads"]["gap"] <= 0.010},
{"name": "gap_biased_1read", "hard": False, "what": "|believed - true| coverage at 1 read",
"target": "<= 0.060", "measured": by_name["biased_1read"]["gap"],
"passed": by_name["biased_1read"]["gap"] <= 0.060},
{"name": "ambiguity_gate_pays", "hard": False,
"what": "litres when the reads disagree, gate on vs off", "target": "on <= off",
"measured": f"{by_name['ambiguous_gate_on']['paint_l']} vs "
f"{by_name['ambiguous_gate_off']['paint_l']}",
"passed": by_name["ambiguous_gate_on"]["paint_l"] <= by_name["ambiguous_gate_off"]["paint_l"]},
{"name": "refusal_costs_nothing", "hard": True, "what": "litres on a wall the gate refused",
"target": "== 0.000", "measured": job["refused_wall_litres"],
"passed": job["refused_wall_litres"] == 0.0},
{"name": "failure_is_isolated", "hard": True, "what": "a refused wall stopped the next one",
"target": "the order carries on",
"measured": "yes" if job["refused_wall_blocks_the_next"] else "no",
"passed": not job["refused_wall_blocks_the_next"]},
{"name": "resume_saves_paint", "hard": True, "what": "paint saved by resuming, nothing repainted",
"target": ">= 0.40 and no repaint",
"measured": f"{job['resume_saves']:.2f} saved, repainted={job['resume_repainted']}",
"passed": job["resume_saves"] >= 0.40 and not job["resume_repainted"]},
{"name": "halt_is_sticky", "hard": True,
"what": "a latched safe-stop halts the order and survives the process",
"target": "safe_stop, then 0.000 L on resume",
"measured": f"{safety['tripped_outcome']} -> {safety['resumed_while_latched']} "
f"({safety['resumed_litres_while_latched']} L)",
"passed": (safety["tripped_outcome"] == "safe_stop"
and safety["resumed_while_latched"] == "safe_stop"
and safety["resumed_litres_while_latched"] == 0.0)},
{"name": "halt_needs_a_name", "hard": True, "what": "a latch clears only for a named operator",
"target": "blank refused, a named operator resumes",
"measured": f"blank_refused={safety['blank_reset_refused']}, "
f"after_reset={safety['after_named_reset']}",
"passed": safety["blank_reset_refused"] and safety["after_named_reset"] != "safe_stop"},
{"name": "deterministic", "hard": True, "what": "the same seeds give the same numbers",
"target": "identical on a second run of the suite", "measured": str(rerun_identical),
"passed": rerun_identical},
]
def render_table(report):
"""The markdown a reader sees, in the same shape as the perception benchmark's table."""
lines = ["| Scenario | Outcome | Rounds | Paint L | Min | True | Believed | Gap | False complete |",
"| --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
for row in report["scenarios"] + report["floor"]:
believed = row.get("coverage_believed")
lines.append(f"| `{row['name']}` | {'/'.join(row['outcomes'])} | {row['rounds']:.1f} | "
f"{row['paint_l']:.3f} | {row['minutes']:.1f} | {row['coverage_true']:.3f} | "
f"{believed:.3f} | {row['gap']:.4f} | {row['false_complete']} |"
if believed is not None else
f"| `{row['name']}` | {'/'.join(row['outcomes'])} | {row['rounds']:.1f} | "
f"{row['paint_l']:.3f} | {row['minutes']:.1f} | {row['coverage_true']:.3f} | "
f"n/a | {row['gap']:.4f} | {row['false_complete']} |")
lines += ["", "| Gate | Target | Measured | Result |", "| --- | --- | --- | --- |"]
for gate in report["gates"]:
lines.append(f"| `{gate['name']}` | {gate['target']} | {gate['measured']} | "
f"{'**pass**' if gate['passed'] else 'FAIL'} |")
return "\n".join(lines) + "\n"
def build_report(seeds=SEEDS, tmp_path=None):
"""Run the whole suite: loop scenarios, the job and safety probes, the floor, then the gates."""
import tempfile
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp_path or tmp)
scenarios, silent, unnamed = run_loop_scenarios(seeds)
again, _, _ = run_loop_scenarios(seeds)
rerun_identical = json.dumps(scenarios, sort_keys=True) == json.dumps(again, sort_keys=True)
job = run_job_probe(tmp_path, JOB_SEEDS)
safety = run_safety_probe(tmp_path, JOB_SEEDS)
floor = run_floor(seeds)
gates = evaluate_gates(scenarios, job, safety, silent, unnamed, rerun_identical)
report = {"suite": SUITE_VERSION, "scope": SCOPE, "fixture": "2.7 m wall plane, 10 mm/unit",
"seeds": list(seeds), "engine": "numpy default_rng, seeded per scenario and round",
"scenarios": scenarios, "job": job, "safety": safety, "floor": floor, "gates": gates,
"hard_gates_passed": sum(1 for g in gates if g["hard"] and g["passed"]),
"hard_gates": sum(1 for g in gates if g["hard"]),
"passed": all(g["passed"] for g in gates if g["hard"])}
report["failed_hard_gates"] = [g["name"] for g in gates if g["hard"] and not g["passed"]]
return report
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0],
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--out", type=Path, help="write the full record here (JSON)")
ap.add_argument("--table", type=Path, help="write the markdown table here")
ap.add_argument("--registry", type=Path, help="append the run to a run_registry.py ledger")
ap.add_argument("--seeds", type=int, default=len(SEEDS), help="how many of the fixed seeds to use")
args = ap.parse_args()
report = build_report(tuple(SEEDS[:args.seeds]))
print(f"{'scenario':<28}{'outcome':<20}{'rounds':>7}{'paint L':>9}{'min':>7}"
f"{'true':>7}{'believed':>9}{'gap':>8}")
for row in report["scenarios"] + report["floor"]:
believed = row.get("coverage_believed")
shown = f"{believed:.3f}" if believed is not None else "n/a"
print(f"{row['name']:<28}{'/'.join(row['outcomes']):<20}{row['rounds']:>7.1f}"
f"{row['paint_l']:>9.3f}{row['minutes']:>7.1f}{row['coverage_true']:>7.3f}"
f"{shown:>9}{row['gap']:>8.4f}")
print()
for gate in report["gates"]:
print(f" [{'pass' if gate['passed'] else 'FAIL'}] {gate['name']:<22}{gate['target']:<36}"
f"{gate['measured']}")
print(f"\nhard gates: {report['hard_gates_passed']}/{report['hard_gates']} "
f"-> {'PASS' if report['passed'] else 'FAIL'}")
if args.out:
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(json.dumps(report, indent=2) + "\n")
print(f"record: {args.out}")
if args.table:
args.table.parent.mkdir(parents=True, exist_ok=True)
args.table.write_text(render_table(report))
print(f"table: {args.table}")
if args.registry:
from run_registry import append
record = append(args.registry, "eval", arch=f"robotics/{SUITE_VERSION}",
config={"seeds": list(report["seeds"]), "scenarios": len(report["scenarios"])},
metrics={"hard_gates_passed": report["hard_gates_passed"],
"hard_gates": report["hard_gates"]},
extra={"suite": SUITE_VERSION, "scope": SCOPE})
print(f"ledger: {args.registry} seq {record['seq']} ({record['hash'][:12]})")
raise SystemExit(0 if report["passed"] else 1)
if __name__ == "__main__":
main()