painting-vision-robotics-kit / eval_surfaces.py
constructelligence's picture
Update code: real-machine G-code adapter, commissioning gate, machine_job runner, pump watchdog firmware, twin completion gate, realism and masking modules (card and weights unchanged)
4eecd03 verified
Raw History Blame Contribute Delete
14.6 kB
#!/usr/bin/env python3
"""Image-level test on real wall photos: does the kit notice wallpaper and bare board?
Scores a checkpoint on the reviewed set built by ``fetch_surface_eval.py``. Each
row of ``labels.csv`` is one photo with a reviewed surface label (``wallpaper``,
``painted``, ``partially_painted``, ``bare_drywall``, ``primer``, ``brick``, ``tile``, ``wood_panel``,
``plaster``, ``painted_over_wallpaper``) or ``exclude``. Nothing here has pixel
masks, so the numbers are about decisions a robot makes per frame:
* **wallpaper** - precision/recall for the model alone (wallpaper fraction of the
wall), the model-free cue alone, and the combined verdict that the safety gate
acts on; false-alarm rate on each look-alike surface; AUROC of each score;
* **paint state** - argmax and confident painted fraction per surface, and how
often a bare or primed wall would be signed off as complete;
* **surface condition** - peeling/flaking/cracked paint and rust found (model,
rust cue, combined verdict), with brick and wood as rust-coloured negatives;
* **wall found** - how often no wall is detected at all.
Usage::
python3 eval_surfaces.py --data surface_eval --checkpoint artifacts/best.pt --report surface_report.json
"""
import argparse
import csv
import json
from pathlib import Path
import cv2
import numpy as np
import torch
from predict import analyze, build_parser
from train import load_checkpoint_model
SURFACES = ("paint_failure", "rust", "stain", "damage",
"wallpaper", "painted_over_wallpaper", "painted", "partially_painted", "bare_drywall", "primer",
"brick", "tile", "wood_panel", "plaster")
WALLPAPER_POSITIVE = {"wallpaper"}
# Painted-over paper is paintable in practice but still worth a look; it is scored
# separately and left out of the wallpaper precision/recall.
WALLPAPER_EXCLUDED = {"painted_over_wallpaper"}
UNFINISHED = {"bare_drywall", "primer", "partially_painted"}
# Drywall is scored where the photo shows it: bare board is positive, other visible
# materials negative. Painted/primed walls hide the substrate, so they are left out.
DRYWALL_POSITIVE = {"bare_drywall"}
DRYWALL_NEGATIVE = {"wallpaper", "brick", "tile", "wood_panel", "plaster"}
# Share of the wall called drywall for a per-image "drywall" verdict. It was never fitted, and both
# splits prefer it lower (val AUROC 0.980, wild 0.895), so 0.20 was costing recall for no precision.
# 0.05 is a deliberate convention (flag a wall as unfinished once ~5% bare board is visible), not a
# fitted value -- see benchmark/records/ft4_operating_point_squeeze.json. Its cost is a higher
# painted-wall drywall false-flag rate (0.53 -> 0.65).
DRYWALL_IMAGE_FRACTION = 0.05 # default; a checkpoint's fitted surface_operating_points.drywall_image_fraction wins
# Surface condition: damaged surfaces are positive; every sound surface (including the
# rust-coloured look-alikes brick and wood) is negative.
DEFECT_POSITIVE = {"paint_failure", "rust", "damage"} # damage = holes, plaster loss, cracks (condition_pool/)
DEFECT_FLAGS = ("defects_detected", "possible_rust")
# Dirt, marks and mould need cleaning, not the paint-failure head: scored on their own, not as negatives.
DEFECT_EXCLUDED = {"stain"}
def auroc(scores, labels):
"""Probability a random positive outscores a random negative (ties count half)."""
scores, labels = np.asarray(scores, float), np.asarray(labels, bool)
pos, neg = scores[labels], scores[~labels]
if not len(pos) or not len(neg):
return None
greater = (pos[:, None] > neg[None, :]).sum() + 0.5 * (pos[:, None] == neg[None, :]).sum()
return round(float(greater / (len(pos) * len(neg))), 4)
def detection(predicted, actual):
predicted, actual = np.asarray(predicted, bool), np.asarray(actual, bool)
tp, fp, fn = int((predicted & actual).sum()), int((predicted & ~actual).sum()), int((~predicted & actual).sum())
precision = tp / (tp + fp) if tp + fp else None
recall = tp / (tp + fn) if tp + fn else None
f1 = 2 * precision * recall / (precision + recall) if precision and recall else None
return {"tp": tp, "fp": fp, "fn": fn, "precision": None if precision is None else round(precision, 4),
"recall": None if recall is None else round(recall, 4), "f1": None if f1 is None else round(f1, 4)}
def summarise(rows, drywall_threshold=DRYWALL_IMAGE_FRACTION):
scored = [r for r in rows if r["wall_pixels"] > 0]
report = {"images": len(rows), "images_with_wall": len(scored),
"per_surface": {}, "wallpaper": {}}
for surface in SURFACES:
group = [r for r in rows if r["label"] == surface]
if not group:
continue
walls = [r for r in group if r["wall_pixels"] > 0]
report["per_surface"][surface] = {
"images": len(group),
"no_wall_rate": round(1 - len(walls) / len(group), 4),
"wallpaper_flag_rate": round(np.mean([r["status"] in ("wallpaper_detected", "possible_wallpaper")
for r in walls]), 4) if walls else None,
"model_wallpaper_rate": round(np.mean([r["model_flag"] for r in walls]), 4) if walls else None,
"mean_drywall_fraction": round(np.mean([r["drywall_fraction"] for r in walls]), 4) if walls else None,
"drywall_flag_rate": round(np.mean([r["drywall_fraction"] >= drywall_threshold for r in walls]), 4)
if walls else None,
"defect_flag_rate": round(np.mean([r["condition_status"] in DEFECT_FLAGS for r in walls]), 4) if walls else None,
"cue_flag_rate": round(np.mean([r["cue_flag"] for r in walls]), 4) if walls else None,
"mean_argmax_painted": round(np.mean([r["argmax_painted"] for r in walls]), 4) if walls else None,
"mean_confident_painted": (round(np.mean([r["confident_painted"] for r in walls]), 4)
if walls and walls[0]["confident_painted"] is not None else None),
"signed_off_complete_rate": round(np.mean([r["completion"] == "complete" for r in walls]), 4) if walls else None,
"argmax_would_complete_rate": round(np.mean([r["argmax_painted"] >= 0.95 for r in walls]), 4) if walls else None,
}
# Wallpaper detection only on frames where a wall was found; a missed wall is reported above.
eligible = [r for r in scored if r["label"] not in WALLPAPER_EXCLUDED]
actual = [r["label"] in WALLPAPER_POSITIVE for r in eligible]
report["wallpaper"] = {
"eligible_images": len(eligible), "positives": int(sum(actual)),
"model": detection([r["model_flag"] for r in eligible], actual),
"cue": detection([r["cue_flag"] for r in eligible], actual),
"combined_verdict": detection([r["status"] in ("wallpaper_detected", "possible_wallpaper") for r in eligible],
actual),
"auroc_model_fraction": auroc([r["model_fraction"] or 0.0 for r in eligible], actual),
"auroc_cue_score": auroc([r["cue_score"] or 0.0 for r in eligible], actual),
}
drywall_rows = [r for r in scored if r["label"] in DRYWALL_POSITIVE | DRYWALL_NEGATIVE]
drywall_actual = [r["label"] in DRYWALL_POSITIVE for r in drywall_rows]
report["drywall"] = {
"eligible_images": len(drywall_rows), "positives": int(sum(drywall_actual)),
"image_fraction_threshold": drywall_threshold,
"model": detection([r["drywall_fraction"] >= drywall_threshold for r in drywall_rows], drywall_actual),
"auroc_drywall_fraction": auroc([r["drywall_fraction"] for r in drywall_rows], drywall_actual),
# On bare board the paint head should say unpainted: mean painted share it still reports.
"bare_board_mean_argmax_painted": (round(np.mean([r["argmax_painted"] for r in drywall_rows
if r["label"] in DRYWALL_POSITIVE]), 4)
if any(drywall_actual) else None),
}
defect_rows = [r for r in scored if r["label"] in SURFACES and r["label"] not in DEFECT_EXCLUDED]
defect_actual = [r["label"] in DEFECT_POSITIVE for r in defect_rows]
report["surface_condition"] = {
"eligible_images": len(defect_rows), "positives": int(sum(defect_actual)),
"combined_verdict": detection([r["condition_status"] in DEFECT_FLAGS for r in defect_rows], defect_actual),
"model": detection([r["condition_status"] == "defects_detected" for r in defect_rows], defect_actual),
"rust_cue": detection([r["rust_cue"] >= 0.5 for r in defect_rows], defect_actual),
"auroc_model_defect_fraction": auroc([r["defect_fraction"] for r in defect_rows], defect_actual),
"auroc_rust_cue": auroc([r["rust_cue"] for r in defect_rows], defect_actual),
# Per type: is the right defect named on its own photos?
"paint_failure_recall": (round(np.mean([r["paint_failure_fraction"] >= 0.005 for r in defect_rows
if r["label"] == "paint_failure"]), 4)
if any(r["label"] == "paint_failure" for r in defect_rows) else None),
"rust_recall": (round(np.mean([r["rust_fraction"] >= 0.005 or r["rust_cue"] >= 0.5 for r in defect_rows
if r["label"] == "rust"]), 4)
if any(r["label"] == "rust" for r in defect_rows) else None),
"not_assessed_images": sum(r["condition_status"] == "not_assessed" for r in defect_rows),
}
unfinished = [r for r in scored if r["label"] in UNFINISHED]
report["unfinished_walls"] = {
"images": len(unfinished),
"argmax_false_complete_rate": round(np.mean([r["argmax_painted"] >= 0.95 for r in unfinished]), 4) if unfinished else None,
"signed_off_false_complete_rate": round(np.mean([r["completion"] == "complete" for r in unfinished]), 4) if unfinished else None,
}
return report
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--data", type=Path, default=Path("surface_eval"))
parser.add_argument("--checkpoint", type=Path, default=Path("artifacts/best.pt"))
parser.add_argument("--labels", type=Path, help="reviewed labels CSV (default DATA/labels.csv)")
parser.add_argument("--report", type=Path, help="write the summary JSON here")
parser.add_argument("--rows", type=Path, help="write per-image results CSV here")
parser.add_argument("--tta", action="store_true")
parser.add_argument("--sam-refine", action="store_true",
help="second-opinion the wallpaper/drywall alarms with SAM 2 (sam_refine.py)")
parser.add_argument("--sam-model", help="SAM 2 checkpoint used by --sam-refine")
parser.add_argument("--fast", action="store_true",
help="skip measurement, site-objects and colour analysis (unused by the surface metrics); faster")
args = parser.parse_args()
# Same thresholds predict.py uses by default.
extra = (["--tta"] if args.tta else []) + (["--sam-refine"] if args.sam_refine else [])
if args.sam_model:
extra += ["--sam-model", args.sam_model]
predict_args = build_parser().parse_args(["unused.jpg"] + extra)
if args.fast: # none of these feed a surface metric; skipping them is a big speed-up
predict_args.measure = False
predict_args.objects = False
predict_args.colors = False
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model, checkpoint = load_checkpoint_model(args.checkpoint, device)
labels = list(csv.DictReader((args.labels or args.data / "labels.csv").open()))
rows = []
for item in labels:
if item["label"] in ("exclude", "") or item["label"] not in SURFACES:
continue
bgr = cv2.imread(str(args.data / "images" / item["file"]))
if bgr is None:
continue
summary, _ = analyze(bgr, model, checkpoint, device, predict_args)
wallpaper = summary["wallpaper"]
cue = (wallpaper.get("cue") or {}).get("score")
model_fraction = wallpaper.get("fraction_of_wall")
completion = summary["completion"]
rows.append({
"file": item["file"], "label": item["label"], "wall_pixels": summary["wall_pixels"],
"status": wallpaper["status"], "model_fraction": model_fraction, "cue_score": cue,
"model_flag": model_fraction is not None and model_fraction >= summary["wallpaper"]["min_fraction"],
"cue_flag": cue is not None and cue >= predict_args.wallpaper_cue_threshold,
"argmax_painted": completion.get("argmax_painted_fraction") or 0.0,
"confident_painted": completion.get("confident_painted_fraction"),
"completion": completion["status"],
"drywall_fraction": summary["drywall"]["fraction_lower_bound"] or 0.0,
"condition_status": summary["surface_condition"]["status"],
"paint_failure_fraction": (summary["surface_condition"].get("paint_failure") or {}).get("fraction_of_wall", 0.0),
"rust_fraction": (summary["surface_condition"].get("rust") or {}).get("fraction_of_wall", 0.0),
"defect_fraction": sum((summary["surface_condition"].get(k) or {}).get("fraction_of_wall", 0.0)
for k in ("paint_failure", "rust")),
"rust_cue": summary["surface_condition"]["cue"]["score"],
})
print(json.dumps(rows[-1]), flush=True)
report = {"checkpoint": str(args.checkpoint), "completion_threshold": checkpoint.get("completion_threshold"),
"material_classes": list(checkpoint.get("material_classes", [])), "tta": args.tta,
"surface_operating_points": checkpoint.get("surface_operating_points"),
**summarise(rows, (checkpoint.get("surface_operating_points") or {}).get("drywall_image_fraction",
DRYWALL_IMAGE_FRACTION))}
print(json.dumps(report, indent=2))
if args.report:
args.report.write_text(json.dumps(report, indent=2) + "\n")
if args.rows and rows:
with args.rows.open("w", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
if __name__ == "__main__":
main()