Hsyshasuzh's picture
download
raw
26.3 kB
"""Measured, CPU-only pretrained baseline on the approved fixed tile records."""
import argparse
import copy
import json
import multiprocessing
import platform
import time
from collections import defaultdict
from pathlib import Path
import numpy as np
from PIL import Image, ImageDraw
from training.model_factory import (
ASSET_BASE, CHECKPOINTS, FRAMEWORK_VERSION, ROOT, detector_transfer_plan,
load_pretrained_detector, load_pretrained_segmenter, public_mapping, sha256,
)
from evaluation.calibrate_thresholds import _validate_selection_records
from evaluation.manifest import DETECTION_CLASS_NAMES, SEGMENTATION_CLASS_NAMES
from evaluation.metrics import _match_class, evaluate_detection
from load_dataset import dataset
from training.runtime import load_runtime_config
def detection_target(sample: dict, provenance: str) -> dict:
x, y = sample["metadata"]["tile_coordinates"][:2]
annotations = copy.deepcopy(sample["annotations"])
for annotation in annotations:
annotation["obb_polygon"] = [
value - (x if index % 2 == 0 else y)
for index, value in enumerate(annotation["obb_polygon"])
]
return {
"sample_id": sample["metadata"]["id"], "split": "val",
"provenance": provenance, "annotations": annotations,
"ignored_annotations": copy.deepcopy(sample["ignored_annotations"]),
}
def selection_thresholds(predictions: list[dict], targets: list[dict]) -> dict:
"""Equivalent F1 search to Task 2, matching once per class, not per threshold.
Confidence filtering removes a suffix of the greedy ordering, so earlier
assignments cannot change. Ignore-only scores are excluded by the matcher.
"""
_validate_selection_records(predictions, targets)
classes = sorted({str(a["category_id"]) for row in targets for a in row["annotations"]})
thresholds, f1, sparse = {}, {}, []
ignored = {row["sample_id"]: row.get("ignored_annotations", []) for row in targets}
for class_id in classes:
by_sample = {
row["sample_id"]: [a for a in row["annotations"] if str(a["category_id"]) == class_id]
for row in targets
}
count = sum(map(len, by_sample.values()))
candidates = [
(row["sample_id"], a) for row in predictions for a in row["annotations"]
if str(a["category_id"]) == class_id
]
scores, _, _ = _match_class(candidates, by_sample, ignored, .5)
score_values = np.array([score for score, _ in scores])
positives = np.array([matched for _, matched in scores], dtype=np.int64)
cumulative = np.cumsum(positives)
scored = []
for threshold in sorted({0.0, *(float(a["confidence"]) for _, a in candidates)}):
kept = int(np.count_nonzero(score_values >= threshold))
tp = int(cumulative[kept - 1]) if kept else 0
scored.append((2 * tp / (count + kept) if count + kept else 0.0, threshold))
best_f1, threshold = max(scored)
thresholds[class_id], f1[class_id] = threshold, best_f1
if count < 2:
sparse.append(class_id)
return {
"thresholds": thresholds, "f1": f1, "provenance": "selection_validation",
"split": "val", "sample_ids": sorted(row["sample_id"] for row in targets),
"sparse_class_ids": sparse,
}
def detection_metrics(predictions: list[dict], targets: list[dict], thresholds: dict) -> dict:
ap = evaluate_detection(predictions, targets)
operating = evaluate_detection(predictions, targets, confidence_thresholds=thresholds)
# Do not publish threshold-truncated AP as if it were the full prediction pool.
for metrics in [operating, *operating["per_class"].values()]:
metrics.pop("ap50", None)
metrics.pop("map50_95", None)
return {"ap": ap, "operating": operating}
def segmentation_counts(raw: np.ndarray, target: np.ndarray, mapping: dict, classes) -> dict:
mapped = np.zeros(raw.shape, dtype=np.uint8)
for source_id, canonical_id in mapping.items():
mapped[raw == source_id] = canonical_id
valid = target != 255
return {
str(class_id): {
"intersection": int(np.count_nonzero(valid & (mapped == class_id) & (target == class_id))),
"union": int(np.count_nonzero(valid & ((mapped == class_id) | (target == class_id)))),
"targets": int(np.count_nonzero(valid & (target == class_id))),
} for class_id in classes
}
def semantic_array(result) -> np.ndarray:
return result.semantic_mask.data.cpu().numpy()
def _preview(sample, raw, mapping, target, path: Path, thresholds=None, prediction_label="pretrained"):
image = Image.fromarray(sample["image"]).convert("RGB")
truth, prediction = image.copy(), image.copy()
colors = {key: ((key * 53) % 200 + 55, (key * 97) % 200 + 55, (key * 139) % 200 + 55)
for key in range(1, 17)}
if sample["metadata"]["task"] == "detection":
for panel, annotations in ((truth, target["annotations"]), (prediction, raw)):
draw = ImageDraw.Draw(panel)
for a in annotations:
class_id = a["category_id"]
if "confidence" in a and a["confidence"] < thresholds.get(str(class_id), 0):
continue
points = a["obb_polygon"]
points = list(zip(points[::2], points[1::2]))
draw.line(points + points[:1], fill=colors[class_id], width=3)
draw.text(points[0], str(class_id), fill=colors[class_id])
else:
mapped = np.zeros(raw.shape, dtype=np.uint8)
for source_id, canonical_id in mapping.items():
mapped[raw == source_id] = canonical_id
for panel, mask in ((truth, target), (prediction, mapped)):
overlay = np.zeros((*mask.shape, 3), dtype=np.uint8)
for class_id, color in colors.items():
overlay[mask == class_id] = color
panel.paste(Image.blend(image, Image.fromarray(overlay), .5))
sheet = Image.new("RGB", (1536, 540), "black")
for index, (label, panel) in enumerate((("input", image), ("truth", truth), (prediction_label, prediction))):
sheet.paste(panel.resize((512, 512)), (index * 512, 28))
ImageDraw.Draw(sheet).text((index * 512 + 8, 8), label, fill="white")
path.parent.mkdir(parents=True, exist_ok=True)
sheet.save(path)
def _validate_sizes(detection_imgsz, segmentation_imgsz):
for value in (detection_imgsz, segmentation_imgsz):
if isinstance(value, bool) or not isinstance(value, int) or value <= 0 or value % 32:
raise ValueError("inference sizes must be positive integer multiples of 32")
def _run_paths(phase, output_dir):
output = Path(output_dir).resolve()
if not output.is_relative_to(ROOT):
raise ValueError("evaluation output must remain within the project")
legacy = ROOT / "evaluation" / "results"
previews = (ROOT / "evaluation" / "predictions" / "baseline"
if output == legacy else output / "predictions" / "baseline")
return output, previews
def _check_fresh_output(phase, output_dir):
output, previews = _run_paths(phase, output_dir)
protected = [output / (phase + suffix) for suffix in
(".json", "_STOP.json", "_INCOMPLETE.json", ".json.partial")]
if previews.exists() or any(path.exists() for path in protected):
raise FileExistsError("evaluation evidence already exists; choose a fresh --output-dir")
return output, previews
def calibration_evidence(targets, selection_rows):
scene_by_id = {row["sample_id"]: row["scene_id"] for row in selection_rows}
classes = sorted({int(a["category_id"]) for row in targets for a in row["annotations"]})
support = {
str(class_id): {
"positive_instances": sum(a["category_id"] == class_id for row in targets for a in row["annotations"]),
"positive_tiles": sum(any(a["category_id"] == class_id for a in row["annotations"]) for row in targets),
"positive_scenes": len({scene_by_id[row["sample_id"]] for row in targets
if any(a["category_id"] == class_id for a in row["annotations"])}),
} for class_id in classes
}
low_support = [key for key, row in support.items()
if row["positive_instances"] < 10 or row["positive_tiles"] < 3]
warnings = [f"Class {key} threshold has limited evidence: {support[key]['positive_instances']} positives "
f"in {support[key]['positive_tiles']} tiles; do not infer stable generalization."
for key in low_support]
return {"per_class": support, "low_support_class_ids": low_support,
"disclosure_rule": "Fewer than 10 positive instances or 3 positive tiles is a disclosure heuristic, not a statistical reliability test; threshold selection is unchanged.",
"warnings": warnings}
def phase_mappings(phase, detector, segmenter):
mappings = {"detection": public_mapping(detector.names, "detection"),
"segmentation": public_mapping(segmenter.names, "segmentation")}
if phase == "BASELINE_ADAPTED":
from training.segmentation_data import CLASS_NAMES, CANONICAL_IDS
if segmenter.names != dict(enumerate(CLASS_NAMES)):
raise ValueError("adapted segmenter does not have the approved seven-class head")
mappings["segmentation"] = dict(enumerate(CANONICAL_IDS))
return mappings
def run_quick_eval(phase, detector, segmenter, manifest: Path, output_dir: Path, *, timeout_seconds=300,
detection_imgsz=1024, segmentation_imgsz=1024, _loading_seconds=0.0) -> dict:
if not 0 < timeout_seconds <= 300:
raise ValueError("approved timeout must be in (0, 300]")
if phase not in {"BASELINE_PRETRAINED", "BASELINE_ADAPTED"}:
raise ValueError("unsupported baseline phase")
_validate_sizes(detection_imgsz, segmentation_imgsz)
output_dir, previews = _check_fresh_output(phase, output_dir)
started = time.perf_counter()
def budget():
if time.perf_counter() - started >= timeout_seconds:
raise TimeoutError("approved quick-evaluation budget exhausted")
fixed = ROOT / "evaluation" / "quick_eval_manifest.json"
if manifest.resolve() != fixed or sha256(manifest) != "0a0934ddd79a7536a5bbca3df85e77cb9110ac5d69afb7f524bb276861e4b9de":
raise ValueError("only the approved fixed manifest is accepted")
spec = json.loads(manifest.read_text())
runtime = load_runtime_config(ROOT / "training" / "configs" / "runtime.json")
import torch
torch.set_num_threads(runtime.num_threads)
torch.manual_seed(runtime.seed)
if (detector.task, segmenter.task) != ("obb", "semantic"):
raise ValueError("requires oriented detection and semantic segmentation checkpoints")
output_dir.mkdir(parents=True, exist_ok=True)
mappings = phase_mappings(phase, detector, segmenter)
adapted = phase == "BASELINE_ADAPTED"
adaptation = {}
if adapted:
from training.model_factory import model_state_sha256
from training.model_factory import DOTA_NAMES
if detector.names != dict(enumerate(DOTA_NAMES)):
raise ValueError("adapted detector does not have the approved16 source classes")
adaptation["adapted_state_sha256"] = {"detection": model_state_sha256(detector.model),
"segmentation": model_state_sha256(segmenter.model)}
pools = defaultdict(list)
timings = []
counts = {str(key): {"intersection": 0, "union": 0, "targets": 0} for key in SEGMENTATION_CLASS_NAMES}
preview_rows = []
previews.mkdir(parents=True, exist_ok=False)
incomplete = output_dir / (phase + "_INCOMPLETE.json")
incomplete.write_text(json.dumps({"status": "INCOMPLETE", "phase": phase}), encoding="utf-8")
raw_dir = previews / "raw"
raw_dir.mkdir()
if adapted:
initial = {}
for task, wrapper in (("detection", detector), ("segmentation", segmenter)):
path = output_dir / (task + "_initial.pt")
with path.open("xb") as handle:
torch.save({"model": copy.deepcopy(wrapper.model).cpu(),
"train_args": {"task": wrapper.task, "imgsz": 1024, "seed": runtime.seed},
"training_performed": False}, handle)
initial[task] = {"path": str(path.relative_to(ROOT)), "sha256": sha256(path)}
adaptation["initial_checkpoints"] = initial
invalid_labeled_semantic_pixels = 0
calibration = None
subset_order = [
("selection_validation", [row for row in spec["selection_validation"] if row["task"] == "detection"]),
("quick_eval", spec["quick_eval"]["detection"] + spec["quick_eval"]["segmentation"]),
]
for provenance, rows in subset_order:
for task in ("detection", "segmentation"):
task_rows = [row for row in rows if row["task"] == task]
if not task_rows:
continue
data = dataset(ROOT / "datasets" / "corpus", task, "val", license_scope="all")
indices = {sample["id"]: index for index, sample in enumerate(data.samples)}
for index, row in enumerate(task_rows):
budget()
sample = data[indices[row["sample_id"]]]
for key in ("image", "labels" if task == "detection" else "mask"):
if row[key] != sample["metadata"][key]:
raise ValueError("manifest asset differs from approved corpus record")
model = detector if task == "detection" else segmenter
tick = time.perf_counter()
result = model.predict(
sample["image"][:, :, ::-1].copy(), device="cpu",
imgsz=detection_imgsz if task == "detection" else segmentation_imgsz,
conf=.001, max_det=300, verbose=False, save=False,
)[0]
elapsed = time.perf_counter() - tick
timings.append({"sample_id": row["sample_id"], "task": task, "provenance": provenance,
"wall_seconds": elapsed, "framework_ms": result.speed})
budget()
if task == "detection":
annotations = []
original = []
for points, cls, confidence in zip(result.obb.xyxyxyxy.cpu().numpy(),
result.obb.cls.cpu().numpy(),
result.obb.conf.cpu().numpy()):
source_id = int(cls)
item = {"source_class_id": source_id, "confidence": float(confidence),
"obb_polygon": points.reshape(-1).tolist()}
original.append(item)
if source_id in mappings[task]:
annotations.append({**item, "category_id": mappings[task][source_id]})
predicted = {**row, "annotations": annotations}
target = detection_target(sample, provenance)
pools[provenance + "_predictions"].append(predicted)
pools[provenance + "_targets"].append(target)
(raw_dir / (row["sample_id"] + ".json")).write_text(json.dumps(original), encoding="utf-8")
raw = annotations
else:
raw = semantic_array(result)
if raw.shape != sample["mask"].shape:
raise RuntimeError("semantic output is not in original tile coordinates")
target = sample["mask"]
if "valid_mask" in sample:
invalid_labeled_semantic_pixels += int(np.count_nonzero((target != 255) & ~sample["valid_mask"]))
np.savez_compressed(raw_dir / (row["sample_id"] + ".npz"), semantic_class_ids=raw)
current = segmentation_counts(raw, target, mappings[task], SEGMENTATION_CLASS_NAMES)
for key in counts:
for field in counts[key]:
counts[key][field] += current[key][field]
if provenance == "quick_eval" and index < 8:
path = previews / (row["sample_id"] + ".jpg")
_preview(sample, raw, mappings[task], target, path,
calibration["thresholds"] if calibration else {},
prediction_label="adapted, untrained" if adapted else "pretrained")
preview_rows.append(str(path.relative_to(ROOT)))
if provenance == "selection_validation":
budget()
calibration = selection_thresholds(pools["selection_validation_predictions"],
pools["selection_validation_targets"])
budget()
metrics_tick = time.perf_counter()
detection = detection_metrics(pools["quick_eval_predictions"], pools["quick_eval_targets"],
calibration["thresholds"])
budget()
mapped_classes = set(mappings["segmentation"].values())
per_class = {
key: {**values, "name": SEGMENTATION_CLASS_NAMES[int(key)],
"available": int(key) in mapped_classes,
"iou": values["intersection"] / values["union"] if values["union"] and int(key) in mapped_classes else None}
for key, values in counts.items()
}
available_ious = [row["iou"] for row in per_class.values() if row["iou"] is not None]
unavailable = [key for key, value in per_class.items() if not value["available"]]
result = {
"phase": phase, "status": "DONE_WITH_CONCERNS", "training_performed": False,
"runtime": {"python": platform.python_version(), "torch": torch.__version__,
"ultralytics": FRAMEWORK_VERSION, "device": "cpu",
"num_threads": runtime.num_threads, "seed": runtime.seed},
"checkpoints": {
name: {"sha256": digest, "url": ASSET_BASE + name,
"license": "Ultralytics AGPL-3.0 / enterprise terms; inspect intended distribution"}
for name, digest in CHECKPOINTS.items()
},
"checkpoint_names": {"detection": detector.names, "segmentation": segmenter.names},
"head_adaptation_applied": adapted,
"future_detector_transfer": detector_transfer_plan(detector.names),
"public_mapping": mappings,
"manifest_sha256": sha256(manifest),
"corpus_content_sha256": json.loads(
(ROOT / "datasets" / "corpus" / "validation.json").read_text())["content_sha256"],
"runtime_config_sha256": sha256(ROOT / "training" / "configs" / "runtime.json"),
"counts": {"quick_detection": 32, "quick_segmentation": 20, "calibration_detection": 68,
"selection_segmentation_reserved_not_used": 21},
"inference_options": {"detection_imgsz": detection_imgsz, "segmentation_imgsz": segmentation_imgsz, "ap_confidence_floor": .001,
"max_det": 300, "input": "original RGB converted to framework BGR"},
"detection": detection, "calibration": calibration,
"calibration_evidence": calibration_evidence(pools["selection_validation_targets"], spec["selection_validation"]),
"segmentation": {"per_class": per_class, "miou_available_aliases": float(np.mean(available_ious)),
"unavailable_class_ids": unavailable, "calibration": "not available: class-map output, no confidence"},
"timing": {"per_record": timings, "inference_seconds": sum(row["wall_seconds"] for row in timings),
"metrics_seconds": time.perf_counter() - metrics_tick,
"total_seconds": time.perf_counter() - started, "timeout_seconds": timeout_seconds,
"loading_seconds": _loading_seconds},
"preview_paths": preview_rows,
"prediction_sha256": {str(path.relative_to(ROOT)): sha256(path)
for path in sorted(previews.rglob("*")) if path.is_file()},
"concerns": [
"Tile-record-disjoint selection, not scene/geographic independence.",
"Bare-ground class 15 has one quick tile and zero selection positives; no stable threshold.",
"ADE tree/field/earth/land are not exact vegetation/agricultural_area/bare_ground/developed_area aliases.",
"Unmapped semantic predictions count as errors on labeled pixels; unavailable classes have null IoU.",
"Available-alias mIoU is not a seven-class OpenEarthMap score.",
"AP uses the separate confidence >=0.001, max_det=300 prediction pool.",
"DOTA imagery academic-only; pretrained license does not grant corpus commercial permission.",
],
}
if adapted:
result.update(adaptation)
result["adaptation_reports"] = {"detection": getattr(detector, "adaptation_report", None),
"segmentation": getattr(segmenter, "adaptation_report", None)}
result["future_detector_transfer"] = result["adaptation_reports"]["detection"]
result["segmentation"]["miou"] = result["segmentation"].pop("miou_available_aliases")
result["segmentation"]["target_policy"] = "original canonical mask, ignore255; same as retained pretrained baseline"
result["segmentation"]["invalid_labeled_pixels"] = invalid_labeled_semantic_pixels
result["concerns"] = [c for c in result["concerns"] if not c.startswith(("ADE ", "Unmapped semantic", "Available-alias"))]
result["concerns"].append("Four semantic rows are fresh; adaptation is not a learning gain. Training comparisons apply the same validity masks before and after learning.")
budget()
partial = output_dir / (phase + ".json.partial")
with partial.open("x", encoding="utf-8") as handle:
handle.write(json.dumps(result, indent=2))
partial.replace(output_dir / (phase + ".json"))
incomplete.unlink()
return result
def _worker(args):
_validate_sizes(args.detection_imgsz, args.segmentation_imgsz)
_check_fresh_output(args.phase, args.output_dir)
try:
cache = ROOT / "training" / "cache" / "checkpoints"
tick = time.perf_counter()
detector = load_pretrained_detector(cache)
segmenter = load_pretrained_segmenter(cache)
if args.phase == "BASELINE_ADAPTED":
from training.detection_loss import build_adapted_detector
from training.segmentation_loss import build_adapted_segmenter
runtime = load_runtime_config(ROOT / "training/configs/runtime.json")
import torch
torch.set_num_threads(runtime.num_threads)
detector.model, detector.adaptation_report = build_adapted_detector(detector, seed=runtime.seed)
segmenter.model, segmenter.adaptation_report = build_adapted_segmenter(segmenter, seed=runtime.seed)
loading = time.perf_counter() - tick
result = run_quick_eval(args.phase, detector, segmenter, Path(args.manifest),
Path(args.output_dir), timeout_seconds=args.timeout_seconds,
detection_imgsz=args.detection_imgsz, segmentation_imgsz=args.segmentation_imgsz,
_loading_seconds=loading)
print(json.dumps({"status": result["status"], "timing": result["timing"]["total_seconds"],
"inference_seconds": result["timing"]["inference_seconds"]}), flush=True)
except FileExistsError:
# Another run may have reserved this output after the initial preflight.
raise
except Exception as error:
_failure(args, f"{type(error).__name__}: {error}")
raise
def _failure(args, diagnosis):
output, _ = _run_paths(args.phase, args.output_dir)
if (output / (args.phase + ".json")).exists():
return
output.mkdir(parents=True, exist_ok=True)
path = output / (args.phase + "_STOP.json")
try:
with path.open("x", encoding="utf-8") as handle:
json.dump({"status": "BLOCKED", "diagnosis": diagnosis,
"timeout_seconds": args.timeout_seconds}, handle, indent=2)
except FileExistsError:
pass
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--phase", choices=["BASELINE_PRETRAINED", "BASELINE_ADAPTED"], required=True)
parser.add_argument("--manifest", default=str(ROOT / "evaluation" / "quick_eval_manifest.json"))
parser.add_argument("--output-dir", default=str(ROOT / "evaluation" / "results"))
parser.add_argument("--timeout-seconds", type=int, default=300)
parser.add_argument("--detection-imgsz", type=int, default=1024)
parser.add_argument("--segmentation-imgsz", type=int, default=1024)
args = parser.parse_args()
if not 0 < args.timeout_seconds <= 300:
parser.error("approved timeout must be in (0, 300]")
try:
_validate_sizes(args.detection_imgsz, args.segmentation_imgsz)
_check_fresh_output(args.phase, args.output_dir)
except (ValueError, FileExistsError) as error:
parser.error(str(error))
process = multiprocessing.get_context("spawn").Process(target=_worker, args=(args,))
process.start()
process.join(args.timeout_seconds)
if process.is_alive():
process.terminate()
process.join()
_failure(args, "hard wall-clock budget exhausted (includes checkpoint loading); worker terminated")
raise SystemExit(1)
raise SystemExit(process.exitcode)
if __name__ == "__main__":
main()

Xet Storage Details

Size:
26.3 kB
·
Xet hash:
ee73ae752895005078f8d39549ddcfe1d7006079251755b4ca0f29150d61df49

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.