Image Segmentation
PyTorch
English
painting-vision-robotics-kit
semantic-segmentation
robotics
edge-ai
construction-ai
autonomous-painting
wall-painting-robot
paint-coverage-estimation
building-facade
drywall
skirting-detection
window-detection
lidar
depth-validation
segformer
deeplabv3
Eval Results (legacy)
Update code: real-machine G-code adapter, commissioning gate, machine_job runner, pump watchdog firmware, twin completion gate, realism and masking modules (card and weights unchanged)
4eecd03 verified Download eval_surfaces.py from constructelligence/painting-vision-robotics-kit: direct link, hf CLI and curl.
- Browser
- Download file 14.6 kB
-
https://huggingface.co/constructelligence/painting-vision-robotics-kit/resolve/main/eval_surfaces.py
- Command line
-
hf download hf://constructelligence/painting-vision-robotics-kit/eval_surfaces.py
-
curl -L -o eval_surfaces.py https://huggingface.co/constructelligence/painting-vision-robotics-kit/resolve/main/eval_surfaces.py
14.6 kB
| #!/usr/bin/env python3 | |
| """Image-level test on real wall photos: does the kit notice wallpaper and bare board? | |
| Scores a checkpoint on the reviewed set built by ``fetch_surface_eval.py``. Each | |
| row of ``labels.csv`` is one photo with a reviewed surface label (``wallpaper``, | |
| ``painted``, ``partially_painted``, ``bare_drywall``, ``primer``, ``brick``, ``tile``, ``wood_panel``, | |
| ``plaster``, ``painted_over_wallpaper``) or ``exclude``. Nothing here has pixel | |
| masks, so the numbers are about decisions a robot makes per frame: | |
| * **wallpaper** - precision/recall for the model alone (wallpaper fraction of the | |
| wall), the model-free cue alone, and the combined verdict that the safety gate | |
| acts on; false-alarm rate on each look-alike surface; AUROC of each score; | |
| * **paint state** - argmax and confident painted fraction per surface, and how | |
| often a bare or primed wall would be signed off as complete; | |
| * **surface condition** - peeling/flaking/cracked paint and rust found (model, | |
| rust cue, combined verdict), with brick and wood as rust-coloured negatives; | |
| * **wall found** - how often no wall is detected at all. | |
| Usage:: | |
| python3 eval_surfaces.py --data surface_eval --checkpoint artifacts/best.pt --report surface_report.json | |
| """ | |
| import argparse | |
| import csv | |
| import json | |
| from pathlib import Path | |
| import cv2 | |
| import numpy as np | |
| import torch | |
| from predict import analyze, build_parser | |
| from train import load_checkpoint_model | |
| SURFACES = ("paint_failure", "rust", "stain", "damage", | |
| "wallpaper", "painted_over_wallpaper", "painted", "partially_painted", "bare_drywall", "primer", | |
| "brick", "tile", "wood_panel", "plaster") | |
| WALLPAPER_POSITIVE = {"wallpaper"} | |
| # Painted-over paper is paintable in practice but still worth a look; it is scored | |
| # separately and left out of the wallpaper precision/recall. | |
| WALLPAPER_EXCLUDED = {"painted_over_wallpaper"} | |
| UNFINISHED = {"bare_drywall", "primer", "partially_painted"} | |
| # Drywall is scored where the photo shows it: bare board is positive, other visible | |
| # materials negative. Painted/primed walls hide the substrate, so they are left out. | |
| DRYWALL_POSITIVE = {"bare_drywall"} | |
| DRYWALL_NEGATIVE = {"wallpaper", "brick", "tile", "wood_panel", "plaster"} | |
| # Share of the wall called drywall for a per-image "drywall" verdict. It was never fitted, and both | |
| # splits prefer it lower (val AUROC 0.980, wild 0.895), so 0.20 was costing recall for no precision. | |
| # 0.05 is a deliberate convention (flag a wall as unfinished once ~5% bare board is visible), not a | |
| # fitted value -- see benchmark/records/ft4_operating_point_squeeze.json. Its cost is a higher | |
| # painted-wall drywall false-flag rate (0.53 -> 0.65). | |
| DRYWALL_IMAGE_FRACTION = 0.05 # default; a checkpoint's fitted surface_operating_points.drywall_image_fraction wins | |
| # Surface condition: damaged surfaces are positive; every sound surface (including the | |
| # rust-coloured look-alikes brick and wood) is negative. | |
| DEFECT_POSITIVE = {"paint_failure", "rust", "damage"} # damage = holes, plaster loss, cracks (condition_pool/) | |
| DEFECT_FLAGS = ("defects_detected", "possible_rust") | |
| # Dirt, marks and mould need cleaning, not the paint-failure head: scored on their own, not as negatives. | |
| DEFECT_EXCLUDED = {"stain"} | |
| def auroc(scores, labels): | |
| """Probability a random positive outscores a random negative (ties count half).""" | |
| scores, labels = np.asarray(scores, float), np.asarray(labels, bool) | |
| pos, neg = scores[labels], scores[~labels] | |
| if not len(pos) or not len(neg): | |
| return None | |
| greater = (pos[:, None] > neg[None, :]).sum() + 0.5 * (pos[:, None] == neg[None, :]).sum() | |
| return round(float(greater / (len(pos) * len(neg))), 4) | |
| def detection(predicted, actual): | |
| predicted, actual = np.asarray(predicted, bool), np.asarray(actual, bool) | |
| tp, fp, fn = int((predicted & actual).sum()), int((predicted & ~actual).sum()), int((~predicted & actual).sum()) | |
| precision = tp / (tp + fp) if tp + fp else None | |
| recall = tp / (tp + fn) if tp + fn else None | |
| f1 = 2 * precision * recall / (precision + recall) if precision and recall else None | |
| return {"tp": tp, "fp": fp, "fn": fn, "precision": None if precision is None else round(precision, 4), | |
| "recall": None if recall is None else round(recall, 4), "f1": None if f1 is None else round(f1, 4)} | |
| def summarise(rows, drywall_threshold=DRYWALL_IMAGE_FRACTION): | |
| scored = [r for r in rows if r["wall_pixels"] > 0] | |
| report = {"images": len(rows), "images_with_wall": len(scored), | |
| "per_surface": {}, "wallpaper": {}} | |
| for surface in SURFACES: | |
| group = [r for r in rows if r["label"] == surface] | |
| if not group: | |
| continue | |
| walls = [r for r in group if r["wall_pixels"] > 0] | |
| report["per_surface"][surface] = { | |
| "images": len(group), | |
| "no_wall_rate": round(1 - len(walls) / len(group), 4), | |
| "wallpaper_flag_rate": round(np.mean([r["status"] in ("wallpaper_detected", "possible_wallpaper") | |
| for r in walls]), 4) if walls else None, | |
| "model_wallpaper_rate": round(np.mean([r["model_flag"] for r in walls]), 4) if walls else None, | |
| "mean_drywall_fraction": round(np.mean([r["drywall_fraction"] for r in walls]), 4) if walls else None, | |
| "drywall_flag_rate": round(np.mean([r["drywall_fraction"] >= drywall_threshold for r in walls]), 4) | |
| if walls else None, | |
| "defect_flag_rate": round(np.mean([r["condition_status"] in DEFECT_FLAGS for r in walls]), 4) if walls else None, | |
| "cue_flag_rate": round(np.mean([r["cue_flag"] for r in walls]), 4) if walls else None, | |
| "mean_argmax_painted": round(np.mean([r["argmax_painted"] for r in walls]), 4) if walls else None, | |
| "mean_confident_painted": (round(np.mean([r["confident_painted"] for r in walls]), 4) | |
| if walls and walls[0]["confident_painted"] is not None else None), | |
| "signed_off_complete_rate": round(np.mean([r["completion"] == "complete" for r in walls]), 4) if walls else None, | |
| "argmax_would_complete_rate": round(np.mean([r["argmax_painted"] >= 0.95 for r in walls]), 4) if walls else None, | |
| } | |
| # Wallpaper detection only on frames where a wall was found; a missed wall is reported above. | |
| eligible = [r for r in scored if r["label"] not in WALLPAPER_EXCLUDED] | |
| actual = [r["label"] in WALLPAPER_POSITIVE for r in eligible] | |
| report["wallpaper"] = { | |
| "eligible_images": len(eligible), "positives": int(sum(actual)), | |
| "model": detection([r["model_flag"] for r in eligible], actual), | |
| "cue": detection([r["cue_flag"] for r in eligible], actual), | |
| "combined_verdict": detection([r["status"] in ("wallpaper_detected", "possible_wallpaper") for r in eligible], | |
| actual), | |
| "auroc_model_fraction": auroc([r["model_fraction"] or 0.0 for r in eligible], actual), | |
| "auroc_cue_score": auroc([r["cue_score"] or 0.0 for r in eligible], actual), | |
| } | |
| drywall_rows = [r for r in scored if r["label"] in DRYWALL_POSITIVE | DRYWALL_NEGATIVE] | |
| drywall_actual = [r["label"] in DRYWALL_POSITIVE for r in drywall_rows] | |
| report["drywall"] = { | |
| "eligible_images": len(drywall_rows), "positives": int(sum(drywall_actual)), | |
| "image_fraction_threshold": drywall_threshold, | |
| "model": detection([r["drywall_fraction"] >= drywall_threshold for r in drywall_rows], drywall_actual), | |
| "auroc_drywall_fraction": auroc([r["drywall_fraction"] for r in drywall_rows], drywall_actual), | |
| # On bare board the paint head should say unpainted: mean painted share it still reports. | |
| "bare_board_mean_argmax_painted": (round(np.mean([r["argmax_painted"] for r in drywall_rows | |
| if r["label"] in DRYWALL_POSITIVE]), 4) | |
| if any(drywall_actual) else None), | |
| } | |
| defect_rows = [r for r in scored if r["label"] in SURFACES and r["label"] not in DEFECT_EXCLUDED] | |
| defect_actual = [r["label"] in DEFECT_POSITIVE for r in defect_rows] | |
| report["surface_condition"] = { | |
| "eligible_images": len(defect_rows), "positives": int(sum(defect_actual)), | |
| "combined_verdict": detection([r["condition_status"] in DEFECT_FLAGS for r in defect_rows], defect_actual), | |
| "model": detection([r["condition_status"] == "defects_detected" for r in defect_rows], defect_actual), | |
| "rust_cue": detection([r["rust_cue"] >= 0.5 for r in defect_rows], defect_actual), | |
| "auroc_model_defect_fraction": auroc([r["defect_fraction"] for r in defect_rows], defect_actual), | |
| "auroc_rust_cue": auroc([r["rust_cue"] for r in defect_rows], defect_actual), | |
| # Per type: is the right defect named on its own photos? | |
| "paint_failure_recall": (round(np.mean([r["paint_failure_fraction"] >= 0.005 for r in defect_rows | |
| if r["label"] == "paint_failure"]), 4) | |
| if any(r["label"] == "paint_failure" for r in defect_rows) else None), | |
| "rust_recall": (round(np.mean([r["rust_fraction"] >= 0.005 or r["rust_cue"] >= 0.5 for r in defect_rows | |
| if r["label"] == "rust"]), 4) | |
| if any(r["label"] == "rust" for r in defect_rows) else None), | |
| "not_assessed_images": sum(r["condition_status"] == "not_assessed" for r in defect_rows), | |
| } | |
| unfinished = [r for r in scored if r["label"] in UNFINISHED] | |
| report["unfinished_walls"] = { | |
| "images": len(unfinished), | |
| "argmax_false_complete_rate": round(np.mean([r["argmax_painted"] >= 0.95 for r in unfinished]), 4) if unfinished else None, | |
| "signed_off_false_complete_rate": round(np.mean([r["completion"] == "complete" for r in unfinished]), 4) if unfinished else None, | |
| } | |
| return report | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument("--data", type=Path, default=Path("surface_eval")) | |
| parser.add_argument("--checkpoint", type=Path, default=Path("artifacts/best.pt")) | |
| parser.add_argument("--labels", type=Path, help="reviewed labels CSV (default DATA/labels.csv)") | |
| parser.add_argument("--report", type=Path, help="write the summary JSON here") | |
| parser.add_argument("--rows", type=Path, help="write per-image results CSV here") | |
| parser.add_argument("--tta", action="store_true") | |
| parser.add_argument("--sam-refine", action="store_true", | |
| help="second-opinion the wallpaper/drywall alarms with SAM 2 (sam_refine.py)") | |
| parser.add_argument("--sam-model", help="SAM 2 checkpoint used by --sam-refine") | |
| parser.add_argument("--fast", action="store_true", | |
| help="skip measurement, site-objects and colour analysis (unused by the surface metrics); faster") | |
| args = parser.parse_args() | |
| # Same thresholds predict.py uses by default. | |
| extra = (["--tta"] if args.tta else []) + (["--sam-refine"] if args.sam_refine else []) | |
| if args.sam_model: | |
| extra += ["--sam-model", args.sam_model] | |
| predict_args = build_parser().parse_args(["unused.jpg"] + extra) | |
| if args.fast: # none of these feed a surface metric; skipping them is a big speed-up | |
| predict_args.measure = False | |
| predict_args.objects = False | |
| predict_args.colors = False | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| model, checkpoint = load_checkpoint_model(args.checkpoint, device) | |
| labels = list(csv.DictReader((args.labels or args.data / "labels.csv").open())) | |
| rows = [] | |
| for item in labels: | |
| if item["label"] in ("exclude", "") or item["label"] not in SURFACES: | |
| continue | |
| bgr = cv2.imread(str(args.data / "images" / item["file"])) | |
| if bgr is None: | |
| continue | |
| summary, _ = analyze(bgr, model, checkpoint, device, predict_args) | |
| wallpaper = summary["wallpaper"] | |
| cue = (wallpaper.get("cue") or {}).get("score") | |
| model_fraction = wallpaper.get("fraction_of_wall") | |
| completion = summary["completion"] | |
| rows.append({ | |
| "file": item["file"], "label": item["label"], "wall_pixels": summary["wall_pixels"], | |
| "status": wallpaper["status"], "model_fraction": model_fraction, "cue_score": cue, | |
| "model_flag": model_fraction is not None and model_fraction >= summary["wallpaper"]["min_fraction"], | |
| "cue_flag": cue is not None and cue >= predict_args.wallpaper_cue_threshold, | |
| "argmax_painted": completion.get("argmax_painted_fraction") or 0.0, | |
| "confident_painted": completion.get("confident_painted_fraction"), | |
| "completion": completion["status"], | |
| "drywall_fraction": summary["drywall"]["fraction_lower_bound"] or 0.0, | |
| "condition_status": summary["surface_condition"]["status"], | |
| "paint_failure_fraction": (summary["surface_condition"].get("paint_failure") or {}).get("fraction_of_wall", 0.0), | |
| "rust_fraction": (summary["surface_condition"].get("rust") or {}).get("fraction_of_wall", 0.0), | |
| "defect_fraction": sum((summary["surface_condition"].get(k) or {}).get("fraction_of_wall", 0.0) | |
| for k in ("paint_failure", "rust")), | |
| "rust_cue": summary["surface_condition"]["cue"]["score"], | |
| }) | |
| print(json.dumps(rows[-1]), flush=True) | |
| report = {"checkpoint": str(args.checkpoint), "completion_threshold": checkpoint.get("completion_threshold"), | |
| "material_classes": list(checkpoint.get("material_classes", [])), "tta": args.tta, | |
| "surface_operating_points": checkpoint.get("surface_operating_points"), | |
| **summarise(rows, (checkpoint.get("surface_operating_points") or {}).get("drywall_image_fraction", | |
| DRYWALL_IMAGE_FRACTION))} | |
| print(json.dumps(report, indent=2)) | |
| if args.report: | |
| args.report.write_text(json.dumps(report, indent=2) + "\n") | |
| if args.rows and rows: | |
| with args.rows.open("w", newline="") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=list(rows[0])) | |
| writer.writeheader() | |
| writer.writerows(rows) | |
| if __name__ == "__main__": | |
| main() | |