#!/usr/bin/env python3 """Image-level test on real wall photos: does the kit notice wallpaper and bare board? Scores a checkpoint on the reviewed set built by ``fetch_surface_eval.py``. Each row of ``labels.csv`` is one photo with a reviewed surface label (``wallpaper``, ``painted``, ``partially_painted``, ``bare_drywall``, ``primer``, ``brick``, ``tile``, ``wood_panel``, ``plaster``, ``painted_over_wallpaper``) or ``exclude``. Nothing here has pixel masks, so the numbers are about decisions a robot makes per frame: * **wallpaper** - precision/recall for the model alone (wallpaper fraction of the wall), the model-free cue alone, and the combined verdict that the safety gate acts on; false-alarm rate on each look-alike surface; AUROC of each score; * **paint state** - argmax and confident painted fraction per surface, and how often a bare or primed wall would be signed off as complete; * **surface condition** - peeling/flaking/cracked paint and rust found (model, rust cue, combined verdict), with brick and wood as rust-coloured negatives; * **wall found** - how often no wall is detected at all. Usage:: python3 eval_surfaces.py --data surface_eval --checkpoint artifacts/best.pt --report surface_report.json """ import argparse import csv import json from pathlib import Path import cv2 import numpy as np import torch from predict import analyze, build_parser from train import load_checkpoint_model SURFACES = ("paint_failure", "rust", "stain", "damage", "wallpaper", "painted_over_wallpaper", "painted", "partially_painted", "bare_drywall", "primer", "brick", "tile", "wood_panel", "plaster") WALLPAPER_POSITIVE = {"wallpaper"} # Painted-over paper is paintable in practice but still worth a look; it is scored # separately and left out of the wallpaper precision/recall. WALLPAPER_EXCLUDED = {"painted_over_wallpaper"} UNFINISHED = {"bare_drywall", "primer", "partially_painted"} # Drywall is scored where the photo shows it: bare board is positive, other visible # materials negative. Painted/primed walls hide the substrate, so they are left out. DRYWALL_POSITIVE = {"bare_drywall"} DRYWALL_NEGATIVE = {"wallpaper", "brick", "tile", "wood_panel", "plaster"} # Share of the wall called drywall for a per-image "drywall" verdict. It was never fitted, and both # splits prefer it lower (val AUROC 0.980, wild 0.895), so 0.20 was costing recall for no precision. # 0.05 is a deliberate convention (flag a wall as unfinished once ~5% bare board is visible), not a # fitted value -- see benchmark/records/ft4_operating_point_squeeze.json. Its cost is a higher # painted-wall drywall false-flag rate (0.53 -> 0.65). DRYWALL_IMAGE_FRACTION = 0.05 # default; a checkpoint's fitted surface_operating_points.drywall_image_fraction wins # Surface condition: damaged surfaces are positive; every sound surface (including the # rust-coloured look-alikes brick and wood) is negative. DEFECT_POSITIVE = {"paint_failure", "rust", "damage"} # damage = holes, plaster loss, cracks (condition_pool/) DEFECT_FLAGS = ("defects_detected", "possible_rust") # Dirt, marks and mould need cleaning, not the paint-failure head: scored on their own, not as negatives. DEFECT_EXCLUDED = {"stain"} def auroc(scores, labels): """Probability a random positive outscores a random negative (ties count half).""" scores, labels = np.asarray(scores, float), np.asarray(labels, bool) pos, neg = scores[labels], scores[~labels] if not len(pos) or not len(neg): return None greater = (pos[:, None] > neg[None, :]).sum() + 0.5 * (pos[:, None] == neg[None, :]).sum() return round(float(greater / (len(pos) * len(neg))), 4) def detection(predicted, actual): predicted, actual = np.asarray(predicted, bool), np.asarray(actual, bool) tp, fp, fn = int((predicted & actual).sum()), int((predicted & ~actual).sum()), int((~predicted & actual).sum()) precision = tp / (tp + fp) if tp + fp else None recall = tp / (tp + fn) if tp + fn else None f1 = 2 * precision * recall / (precision + recall) if precision and recall else None return {"tp": tp, "fp": fp, "fn": fn, "precision": None if precision is None else round(precision, 4), "recall": None if recall is None else round(recall, 4), "f1": None if f1 is None else round(f1, 4)} def summarise(rows, drywall_threshold=DRYWALL_IMAGE_FRACTION): scored = [r for r in rows if r["wall_pixels"] > 0] report = {"images": len(rows), "images_with_wall": len(scored), "per_surface": {}, "wallpaper": {}} for surface in SURFACES: group = [r for r in rows if r["label"] == surface] if not group: continue walls = [r for r in group if r["wall_pixels"] > 0] report["per_surface"][surface] = { "images": len(group), "no_wall_rate": round(1 - len(walls) / len(group), 4), "wallpaper_flag_rate": round(np.mean([r["status"] in ("wallpaper_detected", "possible_wallpaper") for r in walls]), 4) if walls else None, "model_wallpaper_rate": round(np.mean([r["model_flag"] for r in walls]), 4) if walls else None, "mean_drywall_fraction": round(np.mean([r["drywall_fraction"] for r in walls]), 4) if walls else None, "drywall_flag_rate": round(np.mean([r["drywall_fraction"] >= drywall_threshold for r in walls]), 4) if walls else None, "defect_flag_rate": round(np.mean([r["condition_status"] in DEFECT_FLAGS for r in walls]), 4) if walls else None, "cue_flag_rate": round(np.mean([r["cue_flag"] for r in walls]), 4) if walls else None, "mean_argmax_painted": round(np.mean([r["argmax_painted"] for r in walls]), 4) if walls else None, "mean_confident_painted": (round(np.mean([r["confident_painted"] for r in walls]), 4) if walls and walls[0]["confident_painted"] is not None else None), "signed_off_complete_rate": round(np.mean([r["completion"] == "complete" for r in walls]), 4) if walls else None, "argmax_would_complete_rate": round(np.mean([r["argmax_painted"] >= 0.95 for r in walls]), 4) if walls else None, } # Wallpaper detection only on frames where a wall was found; a missed wall is reported above. eligible = [r for r in scored if r["label"] not in WALLPAPER_EXCLUDED] actual = [r["label"] in WALLPAPER_POSITIVE for r in eligible] report["wallpaper"] = { "eligible_images": len(eligible), "positives": int(sum(actual)), "model": detection([r["model_flag"] for r in eligible], actual), "cue": detection([r["cue_flag"] for r in eligible], actual), "combined_verdict": detection([r["status"] in ("wallpaper_detected", "possible_wallpaper") for r in eligible], actual), "auroc_model_fraction": auroc([r["model_fraction"] or 0.0 for r in eligible], actual), "auroc_cue_score": auroc([r["cue_score"] or 0.0 for r in eligible], actual), } drywall_rows = [r for r in scored if r["label"] in DRYWALL_POSITIVE | DRYWALL_NEGATIVE] drywall_actual = [r["label"] in DRYWALL_POSITIVE for r in drywall_rows] report["drywall"] = { "eligible_images": len(drywall_rows), "positives": int(sum(drywall_actual)), "image_fraction_threshold": drywall_threshold, "model": detection([r["drywall_fraction"] >= drywall_threshold for r in drywall_rows], drywall_actual), "auroc_drywall_fraction": auroc([r["drywall_fraction"] for r in drywall_rows], drywall_actual), # On bare board the paint head should say unpainted: mean painted share it still reports. "bare_board_mean_argmax_painted": (round(np.mean([r["argmax_painted"] for r in drywall_rows if r["label"] in DRYWALL_POSITIVE]), 4) if any(drywall_actual) else None), } defect_rows = [r for r in scored if r["label"] in SURFACES and r["label"] not in DEFECT_EXCLUDED] defect_actual = [r["label"] in DEFECT_POSITIVE for r in defect_rows] report["surface_condition"] = { "eligible_images": len(defect_rows), "positives": int(sum(defect_actual)), "combined_verdict": detection([r["condition_status"] in DEFECT_FLAGS for r in defect_rows], defect_actual), "model": detection([r["condition_status"] == "defects_detected" for r in defect_rows], defect_actual), "rust_cue": detection([r["rust_cue"] >= 0.5 for r in defect_rows], defect_actual), "auroc_model_defect_fraction": auroc([r["defect_fraction"] for r in defect_rows], defect_actual), "auroc_rust_cue": auroc([r["rust_cue"] for r in defect_rows], defect_actual), # Per type: is the right defect named on its own photos? "paint_failure_recall": (round(np.mean([r["paint_failure_fraction"] >= 0.005 for r in defect_rows if r["label"] == "paint_failure"]), 4) if any(r["label"] == "paint_failure" for r in defect_rows) else None), "rust_recall": (round(np.mean([r["rust_fraction"] >= 0.005 or r["rust_cue"] >= 0.5 for r in defect_rows if r["label"] == "rust"]), 4) if any(r["label"] == "rust" for r in defect_rows) else None), "not_assessed_images": sum(r["condition_status"] == "not_assessed" for r in defect_rows), } unfinished = [r for r in scored if r["label"] in UNFINISHED] report["unfinished_walls"] = { "images": len(unfinished), "argmax_false_complete_rate": round(np.mean([r["argmax_painted"] >= 0.95 for r in unfinished]), 4) if unfinished else None, "signed_off_false_complete_rate": round(np.mean([r["completion"] == "complete" for r in unfinished]), 4) if unfinished else None, } return report def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--data", type=Path, default=Path("surface_eval")) parser.add_argument("--checkpoint", type=Path, default=Path("artifacts/best.pt")) parser.add_argument("--labels", type=Path, help="reviewed labels CSV (default DATA/labels.csv)") parser.add_argument("--report", type=Path, help="write the summary JSON here") parser.add_argument("--rows", type=Path, help="write per-image results CSV here") parser.add_argument("--tta", action="store_true") parser.add_argument("--sam-refine", action="store_true", help="second-opinion the wallpaper/drywall alarms with SAM 2 (sam_refine.py)") parser.add_argument("--sam-model", help="SAM 2 checkpoint used by --sam-refine") parser.add_argument("--fast", action="store_true", help="skip measurement, site-objects and colour analysis (unused by the surface metrics); faster") args = parser.parse_args() # Same thresholds predict.py uses by default. extra = (["--tta"] if args.tta else []) + (["--sam-refine"] if args.sam_refine else []) if args.sam_model: extra += ["--sam-model", args.sam_model] predict_args = build_parser().parse_args(["unused.jpg"] + extra) if args.fast: # none of these feed a surface metric; skipping them is a big speed-up predict_args.measure = False predict_args.objects = False predict_args.colors = False device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model, checkpoint = load_checkpoint_model(args.checkpoint, device) labels = list(csv.DictReader((args.labels or args.data / "labels.csv").open())) rows = [] for item in labels: if item["label"] in ("exclude", "") or item["label"] not in SURFACES: continue bgr = cv2.imread(str(args.data / "images" / item["file"])) if bgr is None: continue summary, _ = analyze(bgr, model, checkpoint, device, predict_args) wallpaper = summary["wallpaper"] cue = (wallpaper.get("cue") or {}).get("score") model_fraction = wallpaper.get("fraction_of_wall") completion = summary["completion"] rows.append({ "file": item["file"], "label": item["label"], "wall_pixels": summary["wall_pixels"], "status": wallpaper["status"], "model_fraction": model_fraction, "cue_score": cue, "model_flag": model_fraction is not None and model_fraction >= summary["wallpaper"]["min_fraction"], "cue_flag": cue is not None and cue >= predict_args.wallpaper_cue_threshold, "argmax_painted": completion.get("argmax_painted_fraction") or 0.0, "confident_painted": completion.get("confident_painted_fraction"), "completion": completion["status"], "drywall_fraction": summary["drywall"]["fraction_lower_bound"] or 0.0, "condition_status": summary["surface_condition"]["status"], "paint_failure_fraction": (summary["surface_condition"].get("paint_failure") or {}).get("fraction_of_wall", 0.0), "rust_fraction": (summary["surface_condition"].get("rust") or {}).get("fraction_of_wall", 0.0), "defect_fraction": sum((summary["surface_condition"].get(k) or {}).get("fraction_of_wall", 0.0) for k in ("paint_failure", "rust")), "rust_cue": summary["surface_condition"]["cue"]["score"], }) print(json.dumps(rows[-1]), flush=True) report = {"checkpoint": str(args.checkpoint), "completion_threshold": checkpoint.get("completion_threshold"), "material_classes": list(checkpoint.get("material_classes", [])), "tta": args.tta, "surface_operating_points": checkpoint.get("surface_operating_points"), **summarise(rows, (checkpoint.get("surface_operating_points") or {}).get("drywall_image_fraction", DRYWALL_IMAGE_FRACTION))} print(json.dumps(report, indent=2)) if args.report: args.report.write_text(json.dumps(report, indent=2) + "\n") if args.rows and rows: with args.rows.open("w", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=list(rows[0])) writer.writeheader() writer.writerows(rows) if __name__ == "__main__": main()