"""Check a sam3edge model on your own machine: speed, memory, and accuracy on 60 COCO pictures. python validate/validate.py # CPU python validate/validate.py --provider cuda # an NVIDIA GPU (needs onnxruntime-gpu) It downloads the bundles it needs from Hugging Face and 60 pictures from cocodataset.org, runs every bundle in its own process, and writes ``results/-.json`` and a table beside it. With ``--provider cuda`` it also runs each bundle on the CPU and reports how far the GPU's answers are from the CPU's. ``--other`` adds another publisher's ONNX export of the full SAM 3 for comparison. Needs: numpy, pillow, huggingface_hub and onnxruntime (or onnxruntime-gpu); ``tokenizers`` for --other. """ from __future__ import annotations import argparse import json import os import platform import statistics import subprocess import sys import time import urllib.request HERE = os.path.dirname(os.path.abspath(__file__)) KIT = os.path.dirname(HERE) MINI_REPO, MICRO_REPO = "eoinedge/sam3edge-mini", "eoinedge/sam3edge-micro" MINI_BUNDLES = ["quality-336", "quality-672", "fast-1008"] MICRO_BUNDLES = ["fast-672", "fast-1008", "quality-336"] SCORE_FLOOR, THRESHOLDS = 0.05, (0.3, 0.5) TIMING_PICTURES = 3 # --------------------------------------------------------------------------- scoring (60 COCO pictures) def iou(a, b) -> float: x0, y0, x1, y1 = max(a[0], b[0]), max(a[1], b[1]), min(a[2], b[2]), min(a[3], b[3]) inter = max(0.0, x1 - x0) * max(0.0, y1 - y0) union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter return inter / union if union > 0 else 0.0 def match(predictions, truths, crowds, threshold): """Greedy matching at IoU 0.5; a hit on a crowd region is neither right nor wrong.""" used, tp, fp = set(), 0, 0 for score, box in sorted(predictions, key=lambda p: -p[0]): if score < threshold: continue best, best_iou = None, 0.5 for i, truth in enumerate(truths): if i not in used and iou(box, truth) >= best_iou: best, best_iou = i, iou(box, truth) if best is not None: used.add(best) tp += 1 elif not any(iou(box, crowd) >= 0.5 for crowd in crowds): fp += 1 return tp, fp, len(truths) - len(used) def load_pictures(data_dir: str, count: int): from PIL import Image with open(os.path.join(HERE, "coco60.json"), encoding="utf-8") as f: sample = json.load(f) folder = os.path.join(data_dir, "coco-val2017") os.makedirs(folder, exist_ok=True) items = sample["images"][:count] for item in items: path = os.path.join(folder, item["file"]) if not os.path.isfile(path): urllib.request.urlretrieve(item["url"], path) item["image"] = Image.open(path).convert("RGB") return items, sample["expected_f1"] def score(model, items) -> dict: """F1 at each threshold over the pictures: one image pass per picture, then a question per category.""" tally = {t: {"tp": 0, "fp": 0, "fn": 0, "absent_hit": 0} for t in THRESHOLDS} absent_prompts, seconds = 0, 0.0 for item in items: start = time.perf_counter() model.image(item["image"]) answers = {c: model.ask(item["image"], c, SCORE_FLOOR) for c in list(item["present"]) + item["absent"]} seconds += time.perf_counter() - start absent_prompts += len(item["absent"]) for t in THRESHOLDS: for concept, truths in item["present"].items(): tp, fp, fn = match(answers[concept], truths, item["crowds"].get(concept, []), t) tally[t]["tp"] += tp tally[t]["fp"] += fp tally[t]["fn"] += fn for concept in item["absent"]: hits = sum(s >= t for s, _ in answers[concept]) tally[t]["fp"] += hits tally[t]["absent_hit"] += bool(hits) out = {"pictures": len(items), "seconds_per_picture": round(seconds / max(len(items), 1), 2)} for t in THRESHOLDS: tp, fp, fn = tally[t]["tp"], tally[t]["fp"], tally[t]["fn"] precision = tp / (tp + fp) if tp + fp else 0.0 recall = tp / (tp + fn) if tp + fn else 0.0 out[str(t)] = { "precision": round(precision, 3), "recall": round(recall, 3), "f1": round(2 * precision * recall / (precision + recall), 3) if precision + recall else 0.0, "absent_prompts_answered": round(tally[t]["absent_hit"] / max(absent_prompts, 1), 3), } return out # --------------------------------------------------------------------------- the models class MiniModel: def __init__(self, bundle, provider, threads, image_graph=None): sys.path.insert(0, KIT) from sam3edge_mini import Mini self.m = Mini(bundle, threads=threads, provider=provider, force=True, image_graph=image_graph) self.masks = True self.where = {"image": f"{self.m.image_provider} ({self.m.image_graph})", "rest": provider} self.image_key = next(k for k, v in self.m.meta["files"].items() if v == self.m.image_graph and k.startswith("image")) self._features = None def image(self, picture): self._features = self.m.encode_image(picture) def forget_text(self): self.m._phrases.clear() def ask(self, picture, concept, threshold, masks=False): found = self.m.detect(picture, [concept], threshold=threshold, masks=masks, features=self._features) return [(d["score"], d["box_xyxy"]) for d in found] class MicroModel: def __init__(self, bundle, provider, threads, kit): sys.path.insert(0, kit) from sam3edge_micro import Micro self.m = Micro(bundle, threads=threads, provider=provider, force=True) self.masks = False self.where = {"image": getattr(self.m, "image_provider", provider), "rest": provider} self._level = None def image(self, picture): self._level = self.m.encode_image(picture) def forget_text(self): pass # the text was encoded when the bundle was made def ask(self, picture, concept, threshold, masks=False): level = self._level self.m.encode_image = lambda _picture: level # reuse the features for every concept try: found = self.m.detect(picture, [concept], threshold) finally: del self.m.encode_image return [(d["score"], d["box_xyxy"]) for d in found] class OtherModel: """Another publisher's ONNX export of the full SAM 3 in the common three-graph layout (onnx/vision_encoder*.onnx, text_encoder*.onnx, prompt_decoder*.onnx), scored the same way: the same pictures, the same 0.5 IoU suppression, its masks computed and ignored.""" def __init__(self, folder, suffix, provider, threads): import numpy as np import onnxruntime as ort from tokenizers import Tokenizer self.np = np options = ort.SessionOptions() options.log_severity_level = 3 if threads: options.intra_op_num_threads = threads gpu = ("CUDAExecutionProvider", {"arena_extend_strategy": "kSameAsRequested", "cudnn_conv_algo_search": "HEURISTIC"}) if provider == "cuda" and hasattr(ort, "preload_dlls"): try: ort.preload_dlls() except Exception: pass def load(name, where): path = os.path.join(folder, "onnx", f"{name}{suffix}.onnx") providers = [gpu, "CPUExecutionProvider"] if where == "cuda" else ["CPUExecutionProvider"] session = ort.InferenceSession(path, options, providers=providers) if where == "cuda" and "CUDAExecutionProvider" not in session.get_providers(): raise RuntimeError("ONNX Runtime did not load its CUDA provider") return session # the text encoder runs once per phrase: on the CPU, to leave the GPU's memory to the other two self.vision, self.text, self.decoder = load("vision_encoder", provider), load("text_encoder", "cpu"), load("prompt_decoder", provider) self.tokenizer = Tokenizer.from_file(os.path.join(folder, "tokenizer.json")) with open(os.path.join(folder, "tokenizer_config.json"), encoding="utf-8") as f: pad = json.load(f).get("pad_token", "<|endoftext|>") pad = pad["content"] if isinstance(pad, dict) else pad self.tokenizer.enable_padding(length=32, pad_id=self.tokenizer.token_to_id(pad), pad_token=pad) self.tokenizer.enable_truncation(32) self.masks = False self.where = {"image": provider, "rest": f"{provider} (text encoder on the cpu)"} self._phrases, self._fpn, self._size = {}, None, None self._outputs = [o.name for o in self.decoder.get_outputs() if o.name in ("pred_boxes", "pred_logits", "presence_logits")] def image(self, picture): np = self.np from PIL import Image pixels = np.asarray(picture.resize((1008, 1008), Image.BILINEAR), dtype=np.float32) / 255.0 pixels = ((pixels - 0.5) / 0.5).transpose(2, 0, 1)[None] self._fpn, self._size = self.vision.run(None, {"pixel_values": pixels})[:3], picture.size def forget_text(self): self._phrases.clear() def ask(self, picture, concept, threshold, masks=False): np = self.np if concept not in self._phrases: enc = self.tokenizer.encode(concept) ids = np.array([enc.ids], dtype=np.int64) mask = np.array([enc.attention_mask], dtype=np.int64) self._phrases[concept] = (self.text.run(None, {"input_ids": ids, "attention_mask": mask})[0], mask) features, mask = self._phrases[concept] out = dict(zip(self._outputs, self.decoder.run(self._outputs, { "fpn_hidden_state_0": self._fpn[0], "fpn_hidden_state_1": self._fpn[1], "fpn_hidden_state_2": self._fpn[2], "text_features": features, "attention_mask": mask}))) sigmoid = lambda x: 1.0 / (1.0 + np.exp(-x)) # noqa: E731 scores = sigmoid(out["pred_logits"].reshape(-1)) * float(sigmoid(out["presence_logits"].reshape(-1)[0])) width, height = self._size boxes = out["pred_boxes"].reshape(-1, 4) * np.array([width, height, width, height], dtype=np.float32) kept = [] for i in np.argsort(-scores): if scores[i] < threshold: break box = [float(v) for v in boxes[i]] if all(iou(box, other) < 0.5 for _, other in kept): kept.append((float(scores[i]), box)) return kept # --------------------------------------------------------------------------- one model, in its own process def peak_rss_mb(): try: import resource return round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024) except ImportError: try: import psutil info = psutil.Process().memory_info() return round(getattr(info, "peak_wset", info.rss) / 1e6) except Exception: return None def gpu_memory_mb(): """What nvidia-smi says this process holds on the GPU, or None.""" try: text = subprocess.run(["nvidia-smi", "--query-compute-apps=pid,used_memory", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=20).stdout for line in text.splitlines(): pid, used = [part.strip() for part in line.split(",")] if int(pid) == os.getpid(): return int(used) except Exception: pass return None def build(spec: dict, provider: str, threads): if spec["kind"] == "mini": return MiniModel(spec["path"], provider, threads, spec.get("image_graph")) if spec["kind"] == "micro": return MicroModel(spec["path"], provider, threads, spec["kit"]) return OtherModel(spec["path"], spec["suffix"], provider, threads) def time_model(model, items, threshold, runs): """Medians over the timing pictures: the image pass, a first phrase, one more phrase, and masks.""" image_s, first_s, next_s, mask_s, answers = [], [], [], [], [] for item in items[:TIMING_PICTURES]: concepts = (list(item["present"]) + item["absent"])[:2] model.image(item["image"]) model.ask(item["image"], concepts[0], threshold) # warm-up for _ in range(runs): model.forget_text() t0 = time.perf_counter() model.image(item["image"]) t1 = time.perf_counter() first = model.ask(item["image"], concepts[0], threshold) t2 = time.perf_counter() more = model.ask(item["image"], concepts[1], threshold) t3 = time.perf_counter() image_s.append(t1 - t0), first_s.append(t2 - t0), next_s.append(t3 - t2) if model.masks: # the same phrase again, with and without masks: the difference is the mask head t4 = time.perf_counter() model.ask(item["image"], concepts[0], threshold, masks=True) t5 = time.perf_counter() model.ask(item["image"], concepts[0], threshold) mask_s.append((t5 - t4) - (time.perf_counter() - t5)) answers.append({concepts[0]: [round(s, 4) for s, _ in first], concepts[1]: [round(s, 4) for s, _ in more]}) out = {"image_s": round(statistics.median(image_s), 3), "first_phrase_s": round(statistics.median(first_s), 3), "next_phrase_s": round(statistics.median(next_s), 3)} if mask_s: out["masks_extra_s"] = round(max(statistics.median(mask_s), 0.0), 3) return out, answers def run_one(spec: dict, provider: str, args) -> dict: items, _ = load_pictures(args.data, max(args.coco, TIMING_PICTURES)) start = time.perf_counter() model = build(spec, provider, args.threads) result = {"name": spec["name"], "provider": provider, "runs_on": model.where, "image_graph": getattr(model, "image_key", "image"), "load_s": round(time.perf_counter() - start, 2), "threshold": spec["threshold"]} timing, answers = time_model(model, items, spec["threshold"], args.runs) result.update(timing) result["timing_answers"] = answers if args.coco and not spec.get("timing_only"): result["coco"] = score(model, items[: args.coco]) result["peak_memory_mb"] = peak_rss_mb() if provider == "cuda": result["gpu_memory_mb"] = gpu_memory_mb() return result # --------------------------------------------------------------------------- the whole run def environment(provider: str) -> dict: import onnxruntime as ort env = {"machine": platform.node(), "system": f"{platform.system()} {platform.release()}", "python": platform.python_version(), "onnxruntime": ort.__version__, "onnxruntime_providers": ort.get_available_providers(), "cpu_count": os.cpu_count(), "provider": provider} try: with open("/proc/cpuinfo") as f: env["cpu"] = next(line.split(":", 1)[1].strip() for line in f if line.startswith("model name")) with open("/proc/meminfo") as f: env["memory_gb"] = round(int(next(line for line in f if line.startswith("MemTotal")).split()[1]) / 1e6, 1) except Exception: env["cpu"] = platform.processor() try: env["gpu"] = subprocess.run( ["nvidia-smi", "--query-gpu=name,driver_version,memory.total", "--format=csv,noheader"], capture_output=True, text=True, timeout=20).stdout.strip() except Exception: env["gpu"] = None return env OFFLINE = False def fetch(repo: str, patterns, target: str, skip=None) -> str: """Download what is missing of ``patterns`` into ``target``; with --offline, use what is already there.""" if OFFLINE: return target from huggingface_hub import snapshot_download return snapshot_download(repo, allow_patterns=patterns, ignore_patterns=skip, local_dir=target) def child(spec: dict, provider: str, args) -> dict: command = [sys.executable, os.path.abspath(__file__), "--one", json.dumps(spec), "--provider", provider, "--data", args.data, "--coco", str(args.coco), "--runs", str(args.runs)] if args.threads: command += ["--threads", str(args.threads)] done = subprocess.run(command, capture_output=True, text=True) line = next((ln for ln in done.stdout.splitlines() if ln.startswith("RESULT ")), None) if line is None: tail = (done.stderr or done.stdout).strip().splitlines()[-3:] return {"name": spec["name"], "provider": provider, "error": " | ".join(tail)[-600:]} return json.loads(line[7:]) def table(rows, expected) -> str: lines = ["| Model | Runs on | Image pass | First phrase | Next phrase | Masks, extra | Memory | GPU memory | F1 at 0.5 | F1 at 0.3 | Expected F1 |", "|---|---|---|---|---|---|---|---|---|---|---|"] for r in rows: if "error" in r: lines.append(f"| {r['name']} | {r['provider']} | failed: {r['error'][:120]} | | | | | | | | |") continue coco = r.get("coco", {}) want = expected.get(r["name"] + (" float" if r.get("image_graph") == "image_fp32" else ""), {}) f1 = lambda t: f"{coco[t]['f1']:.3f}" if t in coco else "" # noqa: E731 lines.append( f"| {r['name']} | {r['provider']} | {r['image_s']} s | {r['first_phrase_s']} s | {r['next_phrase_s']} s | " f"{str(r['masks_extra_s']) + ' s' if 'masks_extra_s' in r else ''} | {r.get('peak_memory_mb') or ''} MB | " f"{str(r['gpu_memory_mb']) + ' MB' if r.get('gpu_memory_mb') else ''} | {f1('0.5')} | {f1('0.3')} | " f"{want.get('0.5', '')} / {want.get('0.3', '')} |" ) return "\n".join(lines) def main() -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--provider", choices=["cpu", "cuda"], default="cpu") parser.add_argument("--bundles", default=",".join(MINI_BUNDLES), help="sam3edge-mini bundles to run; 'none' for none") parser.add_argument("--micro", default="", help="sam3edge-micro bundles to run too, e.g. fast-672,quality-336") parser.add_argument("--other", default="", help="another publisher's three-graph ONNX export, e.g. tepirale/sam3-ONNX") parser.add_argument("--other-suffix", default=None, help="its precision: '' (fp32), _fp16, _uint8 ... " "(default: fp32 on a GPU, _uint8 on a CPU)") parser.add_argument("--coco", type=int, default=60, help="how many of the 60 pictures to score (0 skips accuracy)") parser.add_argument("--runs", type=int, default=3) parser.add_argument("--threads", type=int) parser.add_argument("--no-cpu-check", action="store_true", help="with cuda: skip the CPU run each GPU run is compared with") parser.add_argument("--offline", action="store_true", help="download no models: use the bundles already in this folder") parser.add_argument("--image-graph", choices=["image", "image_fp32"], help="which form of a bundle's image encoder to run (default: the one that suits the provider)") parser.add_argument("--data", default=os.path.join(KIT, "data")) parser.add_argument("--out", default=os.path.join(KIT, "results")) parser.add_argument("--one", help=argparse.SUPPRESS) args = parser.parse_args() if args.one: print("RESULT " + json.dumps(run_one(json.loads(args.one), args.provider, args)), flush=True) return 0 global OFFLINE OFFLINE = args.offline os.makedirs(args.out, exist_ok=True) os.makedirs(args.data, exist_ok=True) env = environment(args.provider) print(json.dumps(env, indent=1), flush=True) if args.provider == "cuda" and "CUDAExecutionProvider" not in env["onnxruntime_providers"]: print('This onnxruntime has no CUDA provider. Install the GPU build: pip install "onnxruntime-gpu[cuda,cudnn]"') return 2 items, expected = load_pictures(args.data, max(args.coco, TIMING_PICTURES)) print(f"{len(items)} pictures ready", flush=True) specs = [] for name in [b for b in args.bundles.split(",") if b and b != "none"]: # the float image graph (1.8 GB for SAM 3's own encoder) is only of use to a GPU wants_float = args.provider == "cuda" or args.image_graph == "image_fp32" folder = fetch(MINI_REPO, [f"bundles/{name}/*"], KIT, None if wants_float else ["*image-fp32*"]) with open(os.path.join(folder, "bundles", name, "mini.json"), encoding="utf-8") as f: threshold = json.load(f)["threshold"] specs.append({"kind": "mini", "name": f"sam3edge-mini/{name}", "path": os.path.join(folder, "bundles", name), "threshold": threshold, "image_graph": args.image_graph}) if args.micro: kit = fetch(MICRO_REPO, ["sam3edge_micro/*"] + [f"bundles/{n}/*" for n in args.micro.split(",")], os.path.join(KIT, "micro")) for name in args.micro.split(","): with open(os.path.join(kit, "bundles", name, "micro.json"), encoding="utf-8") as f: threshold = json.load(f)["threshold"] specs.append({"kind": "micro", "name": f"sam3edge-micro/{name}", "path": os.path.join(kit, "bundles", name), "kit": kit, "threshold": threshold}) if args.other: suffix = args.other_suffix if args.other_suffix is not None else ("" if args.provider == "cuda" else "_uint8") patterns = [f"onnx/{g}{suffix}.onnx*" for g in ("vision_encoder", "text_encoder", "prompt_decoder")] folder = fetch(args.other, patterns + ["*.json", "*.txt"], os.path.join(KIT, "other", args.other.replace("/", "__"))) specs.append({"kind": "other", "name": f"{args.other} {suffix.strip('_') or 'fp32'} (full SAM 3, 1008 px)", "path": folder, "suffix": suffix, "threshold": 0.5}) rows = [] for spec in specs: print(f"running {spec['name']} on {args.provider} ...", flush=True) row = child(spec, args.provider, args) if args.provider == "cuda" and not args.no_cpu_check and "error" not in row and spec["kind"] != "other": print(" and on the cpu, to compare the answers ...", flush=True) # the same form of the image encoder as the GPU ran, so that only the processor differs cpu = child({**spec, "timing_only": True, "image_graph": row.get("image_graph")}, "cpu", args) if "error" not in cpu: gaps = [abs(a - b) for g, c in zip(row["timing_answers"], cpu["timing_answers"]) for key in g for a, b in zip(g[key], c[key])] row["cpu_check"] = { "same_counts": all(len(g[k]) == len(c[k]) for g, c in zip(row["timing_answers"], cpu["timing_answers"]) for k in g), "largest_score_gap": round(max(gaps), 4) if gaps else 0.0, "cpu_first_phrase_s": cpu["first_phrase_s"], "cpu_image_s": cpu["image_s"], } rows.append(row) print(" " + json.dumps({k: v for k, v in row.items() if k != "timing_answers"}), flush=True) stem = os.path.join(args.out, f"{platform.node() or 'machine'}-{args.provider}") with open(stem + ".json", "w", encoding="utf-8") as f: json.dump({"environment": env, "expected_f1": expected, "rows": rows}, f, indent=1) text = table(rows, expected) with open(stem + ".md", "w", encoding="utf-8") as f: f.write(f"Machine: {env.get('cpu')}; GPU: {env.get('gpu')}; onnxruntime {env['onnxruntime']}; provider {args.provider}\n\n{text}\n") print("\n" + text + f"\n\nwritten: {stem}.json and {stem}.md") return 0 if __name__ == "__main__": sys.exit(main())