Image Segmentation
PyTorch
English
painting-vision-robotics-kit
semantic-segmentation
robotics
edge-ai
construction-ai
autonomous-painting
wall-painting-robot
paint-coverage-estimation
building-facade
drywall
skirting-detection
window-detection
lidar
depth-validation
segformer
deeplabv3
Eval Results (legacy)
File size: 14,557 Bytes
8746bcd 4eecd03 8746bcd 4eecd03 8746bcd 4eecd03 8746bcd 4eecd03 8746bcd 4eecd03 8746bcd 4eecd03 8746bcd 4eecd03 8746bcd 4eecd03 8746bcd 4eecd03 8746bcd 4eecd03 8746bcd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 | #!/usr/bin/env python3
"""Image-level test on real wall photos: does the kit notice wallpaper and bare board?
Scores a checkpoint on the reviewed set built by ``fetch_surface_eval.py``. Each
row of ``labels.csv`` is one photo with a reviewed surface label (``wallpaper``,
``painted``, ``partially_painted``, ``bare_drywall``, ``primer``, ``brick``, ``tile``, ``wood_panel``,
``plaster``, ``painted_over_wallpaper``) or ``exclude``. Nothing here has pixel
masks, so the numbers are about decisions a robot makes per frame:
* **wallpaper** - precision/recall for the model alone (wallpaper fraction of the
wall), the model-free cue alone, and the combined verdict that the safety gate
acts on; false-alarm rate on each look-alike surface; AUROC of each score;
* **paint state** - argmax and confident painted fraction per surface, and how
often a bare or primed wall would be signed off as complete;
* **surface condition** - peeling/flaking/cracked paint and rust found (model,
rust cue, combined verdict), with brick and wood as rust-coloured negatives;
* **wall found** - how often no wall is detected at all.
Usage::
python3 eval_surfaces.py --data surface_eval --checkpoint artifacts/best.pt --report surface_report.json
"""
import argparse
import csv
import json
from pathlib import Path
import cv2
import numpy as np
import torch
from predict import analyze, build_parser
from train import load_checkpoint_model
SURFACES = ("paint_failure", "rust", "stain", "damage",
"wallpaper", "painted_over_wallpaper", "painted", "partially_painted", "bare_drywall", "primer",
"brick", "tile", "wood_panel", "plaster")
WALLPAPER_POSITIVE = {"wallpaper"}
# Painted-over paper is paintable in practice but still worth a look; it is scored
# separately and left out of the wallpaper precision/recall.
WALLPAPER_EXCLUDED = {"painted_over_wallpaper"}
UNFINISHED = {"bare_drywall", "primer", "partially_painted"}
# Drywall is scored where the photo shows it: bare board is positive, other visible
# materials negative. Painted/primed walls hide the substrate, so they are left out.
DRYWALL_POSITIVE = {"bare_drywall"}
DRYWALL_NEGATIVE = {"wallpaper", "brick", "tile", "wood_panel", "plaster"}
# Share of the wall called drywall for a per-image "drywall" verdict. It was never fitted, and both
# splits prefer it lower (val AUROC 0.980, wild 0.895), so 0.20 was costing recall for no precision.
# 0.05 is a deliberate convention (flag a wall as unfinished once ~5% bare board is visible), not a
# fitted value -- see benchmark/records/ft4_operating_point_squeeze.json. Its cost is a higher
# painted-wall drywall false-flag rate (0.53 -> 0.65).
DRYWALL_IMAGE_FRACTION = 0.05 # default; a checkpoint's fitted surface_operating_points.drywall_image_fraction wins
# Surface condition: damaged surfaces are positive; every sound surface (including the
# rust-coloured look-alikes brick and wood) is negative.
DEFECT_POSITIVE = {"paint_failure", "rust", "damage"} # damage = holes, plaster loss, cracks (condition_pool/)
DEFECT_FLAGS = ("defects_detected", "possible_rust")
# Dirt, marks and mould need cleaning, not the paint-failure head: scored on their own, not as negatives.
DEFECT_EXCLUDED = {"stain"}
def auroc(scores, labels):
"""Probability a random positive outscores a random negative (ties count half)."""
scores, labels = np.asarray(scores, float), np.asarray(labels, bool)
pos, neg = scores[labels], scores[~labels]
if not len(pos) or not len(neg):
return None
greater = (pos[:, None] > neg[None, :]).sum() + 0.5 * (pos[:, None] == neg[None, :]).sum()
return round(float(greater / (len(pos) * len(neg))), 4)
def detection(predicted, actual):
predicted, actual = np.asarray(predicted, bool), np.asarray(actual, bool)
tp, fp, fn = int((predicted & actual).sum()), int((predicted & ~actual).sum()), int((~predicted & actual).sum())
precision = tp / (tp + fp) if tp + fp else None
recall = tp / (tp + fn) if tp + fn else None
f1 = 2 * precision * recall / (precision + recall) if precision and recall else None
return {"tp": tp, "fp": fp, "fn": fn, "precision": None if precision is None else round(precision, 4),
"recall": None if recall is None else round(recall, 4), "f1": None if f1 is None else round(f1, 4)}
def summarise(rows, drywall_threshold=DRYWALL_IMAGE_FRACTION):
scored = [r for r in rows if r["wall_pixels"] > 0]
report = {"images": len(rows), "images_with_wall": len(scored),
"per_surface": {}, "wallpaper": {}}
for surface in SURFACES:
group = [r for r in rows if r["label"] == surface]
if not group:
continue
walls = [r for r in group if r["wall_pixels"] > 0]
report["per_surface"][surface] = {
"images": len(group),
"no_wall_rate": round(1 - len(walls) / len(group), 4),
"wallpaper_flag_rate": round(np.mean([r["status"] in ("wallpaper_detected", "possible_wallpaper")
for r in walls]), 4) if walls else None,
"model_wallpaper_rate": round(np.mean([r["model_flag"] for r in walls]), 4) if walls else None,
"mean_drywall_fraction": round(np.mean([r["drywall_fraction"] for r in walls]), 4) if walls else None,
"drywall_flag_rate": round(np.mean([r["drywall_fraction"] >= drywall_threshold for r in walls]), 4)
if walls else None,
"defect_flag_rate": round(np.mean([r["condition_status"] in DEFECT_FLAGS for r in walls]), 4) if walls else None,
"cue_flag_rate": round(np.mean([r["cue_flag"] for r in walls]), 4) if walls else None,
"mean_argmax_painted": round(np.mean([r["argmax_painted"] for r in walls]), 4) if walls else None,
"mean_confident_painted": (round(np.mean([r["confident_painted"] for r in walls]), 4)
if walls and walls[0]["confident_painted"] is not None else None),
"signed_off_complete_rate": round(np.mean([r["completion"] == "complete" for r in walls]), 4) if walls else None,
"argmax_would_complete_rate": round(np.mean([r["argmax_painted"] >= 0.95 for r in walls]), 4) if walls else None,
}
# Wallpaper detection only on frames where a wall was found; a missed wall is reported above.
eligible = [r for r in scored if r["label"] not in WALLPAPER_EXCLUDED]
actual = [r["label"] in WALLPAPER_POSITIVE for r in eligible]
report["wallpaper"] = {
"eligible_images": len(eligible), "positives": int(sum(actual)),
"model": detection([r["model_flag"] for r in eligible], actual),
"cue": detection([r["cue_flag"] for r in eligible], actual),
"combined_verdict": detection([r["status"] in ("wallpaper_detected", "possible_wallpaper") for r in eligible],
actual),
"auroc_model_fraction": auroc([r["model_fraction"] or 0.0 for r in eligible], actual),
"auroc_cue_score": auroc([r["cue_score"] or 0.0 for r in eligible], actual),
}
drywall_rows = [r for r in scored if r["label"] in DRYWALL_POSITIVE | DRYWALL_NEGATIVE]
drywall_actual = [r["label"] in DRYWALL_POSITIVE for r in drywall_rows]
report["drywall"] = {
"eligible_images": len(drywall_rows), "positives": int(sum(drywall_actual)),
"image_fraction_threshold": drywall_threshold,
"model": detection([r["drywall_fraction"] >= drywall_threshold for r in drywall_rows], drywall_actual),
"auroc_drywall_fraction": auroc([r["drywall_fraction"] for r in drywall_rows], drywall_actual),
# On bare board the paint head should say unpainted: mean painted share it still reports.
"bare_board_mean_argmax_painted": (round(np.mean([r["argmax_painted"] for r in drywall_rows
if r["label"] in DRYWALL_POSITIVE]), 4)
if any(drywall_actual) else None),
}
defect_rows = [r for r in scored if r["label"] in SURFACES and r["label"] not in DEFECT_EXCLUDED]
defect_actual = [r["label"] in DEFECT_POSITIVE for r in defect_rows]
report["surface_condition"] = {
"eligible_images": len(defect_rows), "positives": int(sum(defect_actual)),
"combined_verdict": detection([r["condition_status"] in DEFECT_FLAGS for r in defect_rows], defect_actual),
"model": detection([r["condition_status"] == "defects_detected" for r in defect_rows], defect_actual),
"rust_cue": detection([r["rust_cue"] >= 0.5 for r in defect_rows], defect_actual),
"auroc_model_defect_fraction": auroc([r["defect_fraction"] for r in defect_rows], defect_actual),
"auroc_rust_cue": auroc([r["rust_cue"] for r in defect_rows], defect_actual),
# Per type: is the right defect named on its own photos?
"paint_failure_recall": (round(np.mean([r["paint_failure_fraction"] >= 0.005 for r in defect_rows
if r["label"] == "paint_failure"]), 4)
if any(r["label"] == "paint_failure" for r in defect_rows) else None),
"rust_recall": (round(np.mean([r["rust_fraction"] >= 0.005 or r["rust_cue"] >= 0.5 for r in defect_rows
if r["label"] == "rust"]), 4)
if any(r["label"] == "rust" for r in defect_rows) else None),
"not_assessed_images": sum(r["condition_status"] == "not_assessed" for r in defect_rows),
}
unfinished = [r for r in scored if r["label"] in UNFINISHED]
report["unfinished_walls"] = {
"images": len(unfinished),
"argmax_false_complete_rate": round(np.mean([r["argmax_painted"] >= 0.95 for r in unfinished]), 4) if unfinished else None,
"signed_off_false_complete_rate": round(np.mean([r["completion"] == "complete" for r in unfinished]), 4) if unfinished else None,
}
return report
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--data", type=Path, default=Path("surface_eval"))
parser.add_argument("--checkpoint", type=Path, default=Path("artifacts/best.pt"))
parser.add_argument("--labels", type=Path, help="reviewed labels CSV (default DATA/labels.csv)")
parser.add_argument("--report", type=Path, help="write the summary JSON here")
parser.add_argument("--rows", type=Path, help="write per-image results CSV here")
parser.add_argument("--tta", action="store_true")
parser.add_argument("--sam-refine", action="store_true",
help="second-opinion the wallpaper/drywall alarms with SAM 2 (sam_refine.py)")
parser.add_argument("--sam-model", help="SAM 2 checkpoint used by --sam-refine")
parser.add_argument("--fast", action="store_true",
help="skip measurement, site-objects and colour analysis (unused by the surface metrics); faster")
args = parser.parse_args()
# Same thresholds predict.py uses by default.
extra = (["--tta"] if args.tta else []) + (["--sam-refine"] if args.sam_refine else [])
if args.sam_model:
extra += ["--sam-model", args.sam_model]
predict_args = build_parser().parse_args(["unused.jpg"] + extra)
if args.fast: # none of these feed a surface metric; skipping them is a big speed-up
predict_args.measure = False
predict_args.objects = False
predict_args.colors = False
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model, checkpoint = load_checkpoint_model(args.checkpoint, device)
labels = list(csv.DictReader((args.labels or args.data / "labels.csv").open()))
rows = []
for item in labels:
if item["label"] in ("exclude", "") or item["label"] not in SURFACES:
continue
bgr = cv2.imread(str(args.data / "images" / item["file"]))
if bgr is None:
continue
summary, _ = analyze(bgr, model, checkpoint, device, predict_args)
wallpaper = summary["wallpaper"]
cue = (wallpaper.get("cue") or {}).get("score")
model_fraction = wallpaper.get("fraction_of_wall")
completion = summary["completion"]
rows.append({
"file": item["file"], "label": item["label"], "wall_pixels": summary["wall_pixels"],
"status": wallpaper["status"], "model_fraction": model_fraction, "cue_score": cue,
"model_flag": model_fraction is not None and model_fraction >= summary["wallpaper"]["min_fraction"],
"cue_flag": cue is not None and cue >= predict_args.wallpaper_cue_threshold,
"argmax_painted": completion.get("argmax_painted_fraction") or 0.0,
"confident_painted": completion.get("confident_painted_fraction"),
"completion": completion["status"],
"drywall_fraction": summary["drywall"]["fraction_lower_bound"] or 0.0,
"condition_status": summary["surface_condition"]["status"],
"paint_failure_fraction": (summary["surface_condition"].get("paint_failure") or {}).get("fraction_of_wall", 0.0),
"rust_fraction": (summary["surface_condition"].get("rust") or {}).get("fraction_of_wall", 0.0),
"defect_fraction": sum((summary["surface_condition"].get(k) or {}).get("fraction_of_wall", 0.0)
for k in ("paint_failure", "rust")),
"rust_cue": summary["surface_condition"]["cue"]["score"],
})
print(json.dumps(rows[-1]), flush=True)
report = {"checkpoint": str(args.checkpoint), "completion_threshold": checkpoint.get("completion_threshold"),
"material_classes": list(checkpoint.get("material_classes", [])), "tta": args.tta,
"surface_operating_points": checkpoint.get("surface_operating_points"),
**summarise(rows, (checkpoint.get("surface_operating_points") or {}).get("drywall_image_fraction",
DRYWALL_IMAGE_FRACTION))}
print(json.dumps(report, indent=2))
if args.report:
args.report.write_text(json.dumps(report, indent=2) + "\n")
if args.rows and rows:
with args.rows.open("w", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
if __name__ == "__main__":
main()
|