TempuraML Claude Fable 5 commited on
Commit
648016a
·
1 Parent(s): d134447

feat(exp4): tune SAM-3 fusion on smoke evidence — dormant turf is LAWN

Browse files

Measured on a dormant pilot tile: the SAM bare-dirt union overlaps the lawn
union on 42% of the tile with 1% unique -> dirt paints before lawn (lawn wins
overlaps, unique dirt survives). Driveway wins driveway/sidewalk overlaps;
driveway claims on the centerline buffer outside parcels reassign to road
(SAM swallows whole streets). Flag switches from raw grass/dirt overlap
(fires on every leaf-off tile) to unique-dirt >5%. Smoke-mode count message
fixed; --debug-masks added.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Files changed (1) hide show
  1. scripts/exp4/autolabel_sam3.py +27 -10
scripts/exp4/autolabel_sam3.py CHANGED
@@ -65,8 +65,12 @@ PROMPTS = {
65
  "road": ["paved road"],
66
  "vehicle": ["car"],
67
  }
68
- # Paint order: weakest first; later classes overwrite earlier ones where they overlap.
69
- PAINT_ORDER = ["lawn", "dirt", "tree", "driveway", "sidewalk", "road",
 
 
 
 
70
  "pool", "building", "vehicle"]
71
 
72
 
@@ -137,10 +141,14 @@ def fuse(sam: dict[str, np.ndarray], sup: dict[str, np.ndarray],
137
 
138
  masks = dict(sam)
139
  # Road needs the centerline prior or off-parcel evidence; on-parcel "road"
140
- # pixels are usually driveway — reassign them.
141
- road_ok = masks["road"] & (sup["road_prior"] | ~sup["parcel_any"])
142
- masks["driveway"] = masks["driveway"] | (masks["road"] & sup["parcel_any"]
143
- & ~sup["road_prior"])
 
 
 
 
144
  masks["road"] = road_ok
145
  # Building: require SAM + LiDAR agreement for auto-accept; either-alone is a
146
  # conflict zone left as ignore (the reviewer resolves it).
@@ -159,7 +167,9 @@ def fuse(sam: dict[str, np.ndarray], sup: dict[str, np.ndarray],
159
  conflicts = {
160
  "building_iou": round(float(b_agree.sum() / b_union), 3) if b_union else 1.0,
161
  "lawn_on_lidar_roof": round(float(((label == 1) & sup["lidar_building"]).sum() / lawn_px), 3),
162
- "grass_dirt_overlap": round(float((sam["lawn"] & sam["dirt"]).sum() / total), 3),
 
 
163
  "labeled_frac": round(float((label > 0).sum() / total), 3),
164
  "vehicle_frac": round(float((label == 9).sum() / total), 3),
165
  }
@@ -168,8 +178,8 @@ def fuse(sam: dict[str, np.ndarray], sup: dict[str, np.ndarray],
168
  reasons.append("building-mismatch")
169
  if conflicts["lawn_on_lidar_roof"] > 0.05:
170
  reasons.append("lawn-on-roof")
171
- if conflicts["grass_dirt_overlap"] > 0.04:
172
- reasons.append("grass-dirt-ambiguous")
173
  if conflicts["labeled_frac"] < 0.55:
174
  reasons.append("sparse-labels")
175
  conflicts["flag_reasons"] = reasons
@@ -195,6 +205,8 @@ def main() -> None:
195
  ap.add_argument("--data", required=True, help="crops folder from export_training_crops.py")
196
  ap.add_argument("--limit", type=int, default=0)
197
  ap.add_argument("--smoke", action="store_true", help="1 crop + raw API structure dump")
 
 
198
  ap.add_argument("--revision", default=None, help="pin facebook/sam3 revision")
199
  args = ap.parse_args()
200
  data = Path(args.data)
@@ -225,11 +237,11 @@ def main() -> None:
225
 
226
  ids = sorted(p.stem for p in (data / "images").glob("*.jpg"))
227
  todo = [i for i in ids if not (data / "labels" / f"{i}.png").exists()]
 
228
  if args.smoke:
229
  todo = todo[:1]
230
  elif args.limit:
231
  todo = todo[: args.limit]
232
- print(f"crops: {len(ids)} total, {len(ids) - len(todo)} already labeled, {len(todo)} to do")
233
 
234
  flags_path = data / "flags.csv"
235
  done = failed = flagged = 0
@@ -241,6 +253,11 @@ def main() -> None:
241
  .resize((WORK_SIZE, WORK_SIZE), Image.LANCZOS)
242
  sup = load_supervision(data, crop_id, WORK_SIZE)
243
  sam = sam3_class_masks(model, processor, device, image, smoke=args.smoke)
 
 
 
 
 
244
  label, conflicts = fuse(sam, sup, WORK_SIZE)
245
  save_outputs(data, crop_id, image, label, conflicts)
246
  if conflicts["flag_reasons"]:
 
65
  "road": ["paved road"],
66
  "vehicle": ["car"],
67
  }
68
+ # Paint order: weakest first; later classes overwrite earlier ones where they
69
+ # overlap. Dirt paints BEFORE lawn: on leaf-off imagery SAM's "bare dirt" is a
70
+ # near-duplicate of the dormant-lawn mask (measured 42% of a tile overlapping,
71
+ # 1% unique), so lawn wins the overlap and dirt keeps only its unique regions.
72
+ # Sidewalk before driveway for the same reason (driveway wins shared strips).
73
+ PAINT_ORDER = ["dirt", "lawn", "tree", "sidewalk", "driveway", "road",
74
  "pool", "building", "vehicle"]
75
 
76
 
 
141
 
142
  masks = dict(sam)
143
  # Road needs the centerline prior or off-parcel evidence; on-parcel "road"
144
+ # pixels are usually driveway — reassign them. Symmetrically, "driveway"
145
+ # claims on the centerline buffer outside any parcel are actually road
146
+ # (SAM's driveway concept swallows whole streets sometimes).
147
+ street_zone = sup["road_prior"] & ~sup["parcel_any"]
148
+ road_ok = (masks["road"] | (masks["driveway"] & street_zone)) \
149
+ & (sup["road_prior"] | ~sup["parcel_any"])
150
+ masks["driveway"] = (masks["driveway"] | (masks["road"] & sup["parcel_any"]
151
+ & ~sup["road_prior"])) & ~street_zone
152
  masks["road"] = road_ok
153
  # Building: require SAM + LiDAR agreement for auto-accept; either-alone is a
154
  # conflict zone left as ignore (the reviewer resolves it).
 
167
  conflicts = {
168
  "building_iou": round(float(b_agree.sum() / b_union), 3) if b_union else 1.0,
169
  "lawn_on_lidar_roof": round(float(((label == 1) & sup["lidar_building"]).sum() / lawn_px), 3),
170
+ # raw overlap is ~40% on dormant tiles (expected); only UNIQUE dirt is
171
+ # a signal worth human eyes when it's large
172
+ "dirt_unique": round(float((sam["dirt"] & ~sam["lawn"]).sum() / total), 3),
173
  "labeled_frac": round(float((label > 0).sum() / total), 3),
174
  "vehicle_frac": round(float((label == 9).sum() / total), 3),
175
  }
 
178
  reasons.append("building-mismatch")
179
  if conflicts["lawn_on_lidar_roof"] > 0.05:
180
  reasons.append("lawn-on-roof")
181
+ if conflicts["dirt_unique"] > 0.05:
182
+ reasons.append("large-dirt-area")
183
  if conflicts["labeled_frac"] < 0.55:
184
  reasons.append("sparse-labels")
185
  conflicts["flag_reasons"] = reasons
 
205
  ap.add_argument("--data", required=True, help="crops folder from export_training_crops.py")
206
  ap.add_argument("--limit", type=int, default=0)
207
  ap.add_argument("--smoke", action="store_true", help="1 crop + raw API structure dump")
208
+ ap.add_argument("--debug-masks", action="store_true",
209
+ help="save each class's raw SAM union to debug_masks/ (tuning aid)")
210
  ap.add_argument("--revision", default=None, help="pin facebook/sam3 revision")
211
  args = ap.parse_args()
212
  data = Path(args.data)
 
237
 
238
  ids = sorted(p.stem for p in (data / "images").glob("*.jpg"))
239
  todo = [i for i in ids if not (data / "labels" / f"{i}.png").exists()]
240
+ print(f"crops: {len(ids)} total, {len(ids) - len(todo)} already labeled, {len(todo)} to do")
241
  if args.smoke:
242
  todo = todo[:1]
243
  elif args.limit:
244
  todo = todo[: args.limit]
 
245
 
246
  flags_path = data / "flags.csv"
247
  done = failed = flagged = 0
 
253
  .resize((WORK_SIZE, WORK_SIZE), Image.LANCZOS)
254
  sup = load_supervision(data, crop_id, WORK_SIZE)
255
  sam = sam3_class_masks(model, processor, device, image, smoke=args.smoke)
256
+ if args.debug_masks:
257
+ (data / "debug_masks").mkdir(exist_ok=True)
258
+ for cls, m in sam.items():
259
+ Image.fromarray((m * 255).astype(np.uint8)).save(
260
+ data / "debug_masks" / f"{crop_id}_{cls}.png")
261
  label, conflicts = fuse(sam, sup, WORK_SIZE)
262
  save_outputs(data, crop_id, image, label, conflicts)
263
  if conflicts["flag_reasons"]: