sam3edge-mini / validate /validate.py
eoinedge's picture
validation kit: float image encoder, micro bundles, expected values
508733e verified
Raw History Blame Contribute Delete
24 kB
"""Check a sam3edge model on your own machine: speed, memory, and accuracy on 60 COCO pictures.
python validate/validate.py # CPU
python validate/validate.py --provider cuda # an NVIDIA GPU (needs onnxruntime-gpu)
It downloads the bundles it needs from Hugging Face and 60 pictures from cocodataset.org, runs every
bundle in its own process, and writes ``results/<machine>-<provider>.json`` and a table beside it.
With ``--provider cuda`` it also runs each bundle on the CPU and reports how far the GPU's answers are
from the CPU's. ``--other`` adds another publisher's ONNX export of the full SAM 3 for comparison.
Needs: numpy, pillow, huggingface_hub and onnxruntime (or onnxruntime-gpu); ``tokenizers`` for --other.
"""
from __future__ import annotations
import argparse
import json
import os
import platform
import statistics
import subprocess
import sys
import time
import urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
KIT = os.path.dirname(HERE)
MINI_REPO, MICRO_REPO = "eoinedge/sam3edge-mini", "eoinedge/sam3edge-micro"
MINI_BUNDLES = ["quality-336", "quality-672", "fast-1008"]
MICRO_BUNDLES = ["fast-672", "fast-1008", "quality-336"]
SCORE_FLOOR, THRESHOLDS = 0.05, (0.3, 0.5)
TIMING_PICTURES = 3
# --------------------------------------------------------------------------- scoring (60 COCO pictures)
def iou(a, b) -> float:
x0, y0, x1, y1 = max(a[0], b[0]), max(a[1], b[1]), min(a[2], b[2]), min(a[3], b[3])
inter = max(0.0, x1 - x0) * max(0.0, y1 - y0)
union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter
return inter / union if union > 0 else 0.0
def match(predictions, truths, crowds, threshold):
"""Greedy matching at IoU 0.5; a hit on a crowd region is neither right nor wrong."""
used, tp, fp = set(), 0, 0
for score, box in sorted(predictions, key=lambda p: -p[0]):
if score < threshold:
continue
best, best_iou = None, 0.5
for i, truth in enumerate(truths):
if i not in used and iou(box, truth) >= best_iou:
best, best_iou = i, iou(box, truth)
if best is not None:
used.add(best)
tp += 1
elif not any(iou(box, crowd) >= 0.5 for crowd in crowds):
fp += 1
return tp, fp, len(truths) - len(used)
def load_pictures(data_dir: str, count: int):
from PIL import Image
with open(os.path.join(HERE, "coco60.json"), encoding="utf-8") as f:
sample = json.load(f)
folder = os.path.join(data_dir, "coco-val2017")
os.makedirs(folder, exist_ok=True)
items = sample["images"][:count]
for item in items:
path = os.path.join(folder, item["file"])
if not os.path.isfile(path):
urllib.request.urlretrieve(item["url"], path)
item["image"] = Image.open(path).convert("RGB")
return items, sample["expected_f1"]
def score(model, items) -> dict:
"""F1 at each threshold over the pictures: one image pass per picture, then a question per category."""
tally = {t: {"tp": 0, "fp": 0, "fn": 0, "absent_hit": 0} for t in THRESHOLDS}
absent_prompts, seconds = 0, 0.0
for item in items:
start = time.perf_counter()
model.image(item["image"])
answers = {c: model.ask(item["image"], c, SCORE_FLOOR) for c in list(item["present"]) + item["absent"]}
seconds += time.perf_counter() - start
absent_prompts += len(item["absent"])
for t in THRESHOLDS:
for concept, truths in item["present"].items():
tp, fp, fn = match(answers[concept], truths, item["crowds"].get(concept, []), t)
tally[t]["tp"] += tp
tally[t]["fp"] += fp
tally[t]["fn"] += fn
for concept in item["absent"]:
hits = sum(s >= t for s, _ in answers[concept])
tally[t]["fp"] += hits
tally[t]["absent_hit"] += bool(hits)
out = {"pictures": len(items), "seconds_per_picture": round(seconds / max(len(items), 1), 2)}
for t in THRESHOLDS:
tp, fp, fn = tally[t]["tp"], tally[t]["fp"], tally[t]["fn"]
precision = tp / (tp + fp) if tp + fp else 0.0
recall = tp / (tp + fn) if tp + fn else 0.0
out[str(t)] = {
"precision": round(precision, 3), "recall": round(recall, 3),
"f1": round(2 * precision * recall / (precision + recall), 3) if precision + recall else 0.0,
"absent_prompts_answered": round(tally[t]["absent_hit"] / max(absent_prompts, 1), 3),
}
return out
# --------------------------------------------------------------------------- the models
class MiniModel:
def __init__(self, bundle, provider, threads, image_graph=None):
sys.path.insert(0, KIT)
from sam3edge_mini import Mini
self.m = Mini(bundle, threads=threads, provider=provider, force=True, image_graph=image_graph)
self.masks = True
self.where = {"image": f"{self.m.image_provider} ({self.m.image_graph})", "rest": provider}
self.image_key = next(k for k, v in self.m.meta["files"].items() if v == self.m.image_graph and k.startswith("image"))
self._features = None
def image(self, picture):
self._features = self.m.encode_image(picture)
def forget_text(self):
self.m._phrases.clear()
def ask(self, picture, concept, threshold, masks=False):
found = self.m.detect(picture, [concept], threshold=threshold, masks=masks, features=self._features)
return [(d["score"], d["box_xyxy"]) for d in found]
class MicroModel:
def __init__(self, bundle, provider, threads, kit):
sys.path.insert(0, kit)
from sam3edge_micro import Micro
self.m = Micro(bundle, threads=threads, provider=provider, force=True)
self.masks = False
self.where = {"image": getattr(self.m, "image_provider", provider), "rest": provider}
self._level = None
def image(self, picture):
self._level = self.m.encode_image(picture)
def forget_text(self):
pass # the text was encoded when the bundle was made
def ask(self, picture, concept, threshold, masks=False):
level = self._level
self.m.encode_image = lambda _picture: level # reuse the features for every concept
try:
found = self.m.detect(picture, [concept], threshold)
finally:
del self.m.encode_image
return [(d["score"], d["box_xyxy"]) for d in found]
class OtherModel:
"""Another publisher's ONNX export of the full SAM 3 in the common three-graph layout
(onnx/vision_encoder*.onnx, text_encoder*.onnx, prompt_decoder*.onnx), scored the same way:
the same pictures, the same 0.5 IoU suppression, its masks computed and ignored."""
def __init__(self, folder, suffix, provider, threads):
import numpy as np
import onnxruntime as ort
from tokenizers import Tokenizer
self.np = np
options = ort.SessionOptions()
options.log_severity_level = 3
if threads:
options.intra_op_num_threads = threads
gpu = ("CUDAExecutionProvider", {"arena_extend_strategy": "kSameAsRequested",
"cudnn_conv_algo_search": "HEURISTIC"})
if provider == "cuda" and hasattr(ort, "preload_dlls"):
try:
ort.preload_dlls()
except Exception:
pass
def load(name, where):
path = os.path.join(folder, "onnx", f"{name}{suffix}.onnx")
providers = [gpu, "CPUExecutionProvider"] if where == "cuda" else ["CPUExecutionProvider"]
session = ort.InferenceSession(path, options, providers=providers)
if where == "cuda" and "CUDAExecutionProvider" not in session.get_providers():
raise RuntimeError("ONNX Runtime did not load its CUDA provider")
return session
# the text encoder runs once per phrase: on the CPU, to leave the GPU's memory to the other two
self.vision, self.text, self.decoder = load("vision_encoder", provider), load("text_encoder", "cpu"), load("prompt_decoder", provider)
self.tokenizer = Tokenizer.from_file(os.path.join(folder, "tokenizer.json"))
with open(os.path.join(folder, "tokenizer_config.json"), encoding="utf-8") as f:
pad = json.load(f).get("pad_token", "<|endoftext|>")
pad = pad["content"] if isinstance(pad, dict) else pad
self.tokenizer.enable_padding(length=32, pad_id=self.tokenizer.token_to_id(pad), pad_token=pad)
self.tokenizer.enable_truncation(32)
self.masks = False
self.where = {"image": provider, "rest": f"{provider} (text encoder on the cpu)"}
self._phrases, self._fpn, self._size = {}, None, None
self._outputs = [o.name for o in self.decoder.get_outputs() if o.name in ("pred_boxes", "pred_logits", "presence_logits")]
def image(self, picture):
np = self.np
from PIL import Image
pixels = np.asarray(picture.resize((1008, 1008), Image.BILINEAR), dtype=np.float32) / 255.0
pixels = ((pixels - 0.5) / 0.5).transpose(2, 0, 1)[None]
self._fpn, self._size = self.vision.run(None, {"pixel_values": pixels})[:3], picture.size
def forget_text(self):
self._phrases.clear()
def ask(self, picture, concept, threshold, masks=False):
np = self.np
if concept not in self._phrases:
enc = self.tokenizer.encode(concept)
ids = np.array([enc.ids], dtype=np.int64)
mask = np.array([enc.attention_mask], dtype=np.int64)
self._phrases[concept] = (self.text.run(None, {"input_ids": ids, "attention_mask": mask})[0], mask)
features, mask = self._phrases[concept]
out = dict(zip(self._outputs, self.decoder.run(self._outputs, {
"fpn_hidden_state_0": self._fpn[0], "fpn_hidden_state_1": self._fpn[1], "fpn_hidden_state_2": self._fpn[2],
"text_features": features, "attention_mask": mask})))
sigmoid = lambda x: 1.0 / (1.0 + np.exp(-x)) # noqa: E731
scores = sigmoid(out["pred_logits"].reshape(-1)) * float(sigmoid(out["presence_logits"].reshape(-1)[0]))
width, height = self._size
boxes = out["pred_boxes"].reshape(-1, 4) * np.array([width, height, width, height], dtype=np.float32)
kept = []
for i in np.argsort(-scores):
if scores[i] < threshold:
break
box = [float(v) for v in boxes[i]]
if all(iou(box, other) < 0.5 for _, other in kept):
kept.append((float(scores[i]), box))
return kept
# --------------------------------------------------------------------------- one model, in its own process
def peak_rss_mb():
try:
import resource
return round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)
except ImportError:
try:
import psutil
info = psutil.Process().memory_info()
return round(getattr(info, "peak_wset", info.rss) / 1e6)
except Exception:
return None
def gpu_memory_mb():
"""What nvidia-smi says this process holds on the GPU, or None."""
try:
text = subprocess.run(["nvidia-smi", "--query-compute-apps=pid,used_memory", "--format=csv,noheader,nounits"],
capture_output=True, text=True, timeout=20).stdout
for line in text.splitlines():
pid, used = [part.strip() for part in line.split(",")]
if int(pid) == os.getpid():
return int(used)
except Exception:
pass
return None
def build(spec: dict, provider: str, threads):
if spec["kind"] == "mini":
return MiniModel(spec["path"], provider, threads, spec.get("image_graph"))
if spec["kind"] == "micro":
return MicroModel(spec["path"], provider, threads, spec["kit"])
return OtherModel(spec["path"], spec["suffix"], provider, threads)
def time_model(model, items, threshold, runs):
"""Medians over the timing pictures: the image pass, a first phrase, one more phrase, and masks."""
image_s, first_s, next_s, mask_s, answers = [], [], [], [], []
for item in items[:TIMING_PICTURES]:
concepts = (list(item["present"]) + item["absent"])[:2]
model.image(item["image"])
model.ask(item["image"], concepts[0], threshold) # warm-up
for _ in range(runs):
model.forget_text()
t0 = time.perf_counter()
model.image(item["image"])
t1 = time.perf_counter()
first = model.ask(item["image"], concepts[0], threshold)
t2 = time.perf_counter()
more = model.ask(item["image"], concepts[1], threshold)
t3 = time.perf_counter()
image_s.append(t1 - t0), first_s.append(t2 - t0), next_s.append(t3 - t2)
if model.masks: # the same phrase again, with and without masks: the difference is the mask head
t4 = time.perf_counter()
model.ask(item["image"], concepts[0], threshold, masks=True)
t5 = time.perf_counter()
model.ask(item["image"], concepts[0], threshold)
mask_s.append((t5 - t4) - (time.perf_counter() - t5))
answers.append({concepts[0]: [round(s, 4) for s, _ in first], concepts[1]: [round(s, 4) for s, _ in more]})
out = {"image_s": round(statistics.median(image_s), 3), "first_phrase_s": round(statistics.median(first_s), 3),
"next_phrase_s": round(statistics.median(next_s), 3)}
if mask_s:
out["masks_extra_s"] = round(max(statistics.median(mask_s), 0.0), 3)
return out, answers
def run_one(spec: dict, provider: str, args) -> dict:
items, _ = load_pictures(args.data, max(args.coco, TIMING_PICTURES))
start = time.perf_counter()
model = build(spec, provider, args.threads)
result = {"name": spec["name"], "provider": provider, "runs_on": model.where,
"image_graph": getattr(model, "image_key", "image"),
"load_s": round(time.perf_counter() - start, 2), "threshold": spec["threshold"]}
timing, answers = time_model(model, items, spec["threshold"], args.runs)
result.update(timing)
result["timing_answers"] = answers
if args.coco and not spec.get("timing_only"):
result["coco"] = score(model, items[: args.coco])
result["peak_memory_mb"] = peak_rss_mb()
if provider == "cuda":
result["gpu_memory_mb"] = gpu_memory_mb()
return result
# --------------------------------------------------------------------------- the whole run
def environment(provider: str) -> dict:
import onnxruntime as ort
env = {"machine": platform.node(), "system": f"{platform.system()} {platform.release()}",
"python": platform.python_version(), "onnxruntime": ort.__version__,
"onnxruntime_providers": ort.get_available_providers(), "cpu_count": os.cpu_count(), "provider": provider}
try:
with open("/proc/cpuinfo") as f:
env["cpu"] = next(line.split(":", 1)[1].strip() for line in f if line.startswith("model name"))
with open("/proc/meminfo") as f:
env["memory_gb"] = round(int(next(line for line in f if line.startswith("MemTotal")).split()[1]) / 1e6, 1)
except Exception:
env["cpu"] = platform.processor()
try:
env["gpu"] = subprocess.run(
["nvidia-smi", "--query-gpu=name,driver_version,memory.total", "--format=csv,noheader"],
capture_output=True, text=True, timeout=20).stdout.strip()
except Exception:
env["gpu"] = None
return env
OFFLINE = False
def fetch(repo: str, patterns, target: str, skip=None) -> str:
"""Download what is missing of ``patterns`` into ``target``; with --offline, use what is already there."""
if OFFLINE:
return target
from huggingface_hub import snapshot_download
return snapshot_download(repo, allow_patterns=patterns, ignore_patterns=skip, local_dir=target)
def child(spec: dict, provider: str, args) -> dict:
command = [sys.executable, os.path.abspath(__file__), "--one", json.dumps(spec), "--provider", provider,
"--data", args.data, "--coco", str(args.coco), "--runs", str(args.runs)]
if args.threads:
command += ["--threads", str(args.threads)]
done = subprocess.run(command, capture_output=True, text=True)
line = next((ln for ln in done.stdout.splitlines() if ln.startswith("RESULT ")), None)
if line is None:
tail = (done.stderr or done.stdout).strip().splitlines()[-3:]
return {"name": spec["name"], "provider": provider, "error": " | ".join(tail)[-600:]}
return json.loads(line[7:])
def table(rows, expected) -> str:
lines = ["| Model | Runs on | Image pass | First phrase | Next phrase | Masks, extra | Memory | GPU memory | F1 at 0.5 | F1 at 0.3 | Expected F1 |",
"|---|---|---|---|---|---|---|---|---|---|---|"]
for r in rows:
if "error" in r:
lines.append(f"| {r['name']} | {r['provider']} | failed: {r['error'][:120]} | | | | | | | | |")
continue
coco = r.get("coco", {})
want = expected.get(r["name"] + (" float" if r.get("image_graph") == "image_fp32" else ""), {})
f1 = lambda t: f"{coco[t]['f1']:.3f}" if t in coco else "" # noqa: E731
lines.append(
f"| {r['name']} | {r['provider']} | {r['image_s']} s | {r['first_phrase_s']} s | {r['next_phrase_s']} s | "
f"{str(r['masks_extra_s']) + ' s' if 'masks_extra_s' in r else ''} | {r.get('peak_memory_mb') or ''} MB | "
f"{str(r['gpu_memory_mb']) + ' MB' if r.get('gpu_memory_mb') else ''} | {f1('0.5')} | {f1('0.3')} | "
f"{want.get('0.5', '')} / {want.get('0.3', '')} |"
)
return "\n".join(lines)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--provider", choices=["cpu", "cuda"], default="cpu")
parser.add_argument("--bundles", default=",".join(MINI_BUNDLES), help="sam3edge-mini bundles to run; 'none' for none")
parser.add_argument("--micro", default="", help="sam3edge-micro bundles to run too, e.g. fast-672,quality-336")
parser.add_argument("--other", default="", help="another publisher's three-graph ONNX export, e.g. tepirale/sam3-ONNX")
parser.add_argument("--other-suffix", default=None, help="its precision: '' (fp32), _fp16, _uint8 ... "
"(default: fp32 on a GPU, _uint8 on a CPU)")
parser.add_argument("--coco", type=int, default=60, help="how many of the 60 pictures to score (0 skips accuracy)")
parser.add_argument("--runs", type=int, default=3)
parser.add_argument("--threads", type=int)
parser.add_argument("--no-cpu-check", action="store_true", help="with cuda: skip the CPU run each GPU run is compared with")
parser.add_argument("--offline", action="store_true", help="download no models: use the bundles already in this folder")
parser.add_argument("--image-graph", choices=["image", "image_fp32"],
help="which form of a bundle's image encoder to run (default: the one that suits the provider)")
parser.add_argument("--data", default=os.path.join(KIT, "data"))
parser.add_argument("--out", default=os.path.join(KIT, "results"))
parser.add_argument("--one", help=argparse.SUPPRESS)
args = parser.parse_args()
if args.one:
print("RESULT " + json.dumps(run_one(json.loads(args.one), args.provider, args)), flush=True)
return 0
global OFFLINE
OFFLINE = args.offline
os.makedirs(args.out, exist_ok=True)
os.makedirs(args.data, exist_ok=True)
env = environment(args.provider)
print(json.dumps(env, indent=1), flush=True)
if args.provider == "cuda" and "CUDAExecutionProvider" not in env["onnxruntime_providers"]:
print('This onnxruntime has no CUDA provider. Install the GPU build: pip install "onnxruntime-gpu[cuda,cudnn]"')
return 2
items, expected = load_pictures(args.data, max(args.coco, TIMING_PICTURES))
print(f"{len(items)} pictures ready", flush=True)
specs = []
for name in [b for b in args.bundles.split(",") if b and b != "none"]:
# the float image graph (1.8 GB for SAM 3's own encoder) is only of use to a GPU
wants_float = args.provider == "cuda" or args.image_graph == "image_fp32"
folder = fetch(MINI_REPO, [f"bundles/{name}/*"], KIT, None if wants_float else ["*image-fp32*"])
with open(os.path.join(folder, "bundles", name, "mini.json"), encoding="utf-8") as f:
threshold = json.load(f)["threshold"]
specs.append({"kind": "mini", "name": f"sam3edge-mini/{name}", "path": os.path.join(folder, "bundles", name),
"threshold": threshold, "image_graph": args.image_graph})
if args.micro:
kit = fetch(MICRO_REPO, ["sam3edge_micro/*"] + [f"bundles/{n}/*" for n in args.micro.split(",")],
os.path.join(KIT, "micro"))
for name in args.micro.split(","):
with open(os.path.join(kit, "bundles", name, "micro.json"), encoding="utf-8") as f:
threshold = json.load(f)["threshold"]
specs.append({"kind": "micro", "name": f"sam3edge-micro/{name}", "path": os.path.join(kit, "bundles", name),
"kit": kit, "threshold": threshold})
if args.other:
suffix = args.other_suffix if args.other_suffix is not None else ("" if args.provider == "cuda" else "_uint8")
patterns = [f"onnx/{g}{suffix}.onnx*" for g in ("vision_encoder", "text_encoder", "prompt_decoder")]
folder = fetch(args.other, patterns + ["*.json", "*.txt"], os.path.join(KIT, "other", args.other.replace("/", "__")))
specs.append({"kind": "other", "name": f"{args.other} {suffix.strip('_') or 'fp32'} (full SAM 3, 1008 px)",
"path": folder, "suffix": suffix, "threshold": 0.5})
rows = []
for spec in specs:
print(f"running {spec['name']} on {args.provider} ...", flush=True)
row = child(spec, args.provider, args)
if args.provider == "cuda" and not args.no_cpu_check and "error" not in row and spec["kind"] != "other":
print(" and on the cpu, to compare the answers ...", flush=True)
# the same form of the image encoder as the GPU ran, so that only the processor differs
cpu = child({**spec, "timing_only": True, "image_graph": row.get("image_graph")}, "cpu", args)
if "error" not in cpu:
gaps = [abs(a - b) for g, c in zip(row["timing_answers"], cpu["timing_answers"])
for key in g for a, b in zip(g[key], c[key])]
row["cpu_check"] = {
"same_counts": all(len(g[k]) == len(c[k]) for g, c in zip(row["timing_answers"], cpu["timing_answers"]) for k in g),
"largest_score_gap": round(max(gaps), 4) if gaps else 0.0,
"cpu_first_phrase_s": cpu["first_phrase_s"], "cpu_image_s": cpu["image_s"],
}
rows.append(row)
print(" " + json.dumps({k: v for k, v in row.items() if k != "timing_answers"}), flush=True)
stem = os.path.join(args.out, f"{platform.node() or 'machine'}-{args.provider}")
with open(stem + ".json", "w", encoding="utf-8") as f:
json.dump({"environment": env, "expected_f1": expected, "rows": rows}, f, indent=1)
text = table(rows, expected)
with open(stem + ".md", "w", encoding="utf-8") as f:
f.write(f"Machine: {env.get('cpu')}; GPU: {env.get('gpu')}; onnxruntime {env['onnxruntime']}; provider {args.provider}\n\n{text}\n")
print("\n" + text + f"\n\nwritten: {stem}.json and {stem}.md")
return 0
if __name__ == "__main__":
sys.exit(main())