Download validate/validate.py from eoinedge/sam3edge-mini: direct link, hf CLI and curl.
- Browser
- Download file 24 kB
-
https://huggingface.co/eoinedge/sam3edge-mini/resolve/main/validate/validate.py
- Command line
-
hf download hf://eoinedge/sam3edge-mini/validate/validate.py
-
curl -L -o validate.py https://huggingface.co/eoinedge/sam3edge-mini/resolve/main/validate/validate.py
24 kB
| """Check a sam3edge model on your own machine: speed, memory, and accuracy on 60 COCO pictures. | |
| python validate/validate.py # CPU | |
| python validate/validate.py --provider cuda # an NVIDIA GPU (needs onnxruntime-gpu) | |
| It downloads the bundles it needs from Hugging Face and 60 pictures from cocodataset.org, runs every | |
| bundle in its own process, and writes ``results/<machine>-<provider>.json`` and a table beside it. | |
| With ``--provider cuda`` it also runs each bundle on the CPU and reports how far the GPU's answers are | |
| from the CPU's. ``--other`` adds another publisher's ONNX export of the full SAM 3 for comparison. | |
| Needs: numpy, pillow, huggingface_hub and onnxruntime (or onnxruntime-gpu); ``tokenizers`` for --other. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import platform | |
| import statistics | |
| import subprocess | |
| import sys | |
| import time | |
| import urllib.request | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| KIT = os.path.dirname(HERE) | |
| MINI_REPO, MICRO_REPO = "eoinedge/sam3edge-mini", "eoinedge/sam3edge-micro" | |
| MINI_BUNDLES = ["quality-336", "quality-672", "fast-1008"] | |
| MICRO_BUNDLES = ["fast-672", "fast-1008", "quality-336"] | |
| SCORE_FLOOR, THRESHOLDS = 0.05, (0.3, 0.5) | |
| TIMING_PICTURES = 3 | |
| # --------------------------------------------------------------------------- scoring (60 COCO pictures) | |
| def iou(a, b) -> float: | |
| x0, y0, x1, y1 = max(a[0], b[0]), max(a[1], b[1]), min(a[2], b[2]), min(a[3], b[3]) | |
| inter = max(0.0, x1 - x0) * max(0.0, y1 - y0) | |
| union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter | |
| return inter / union if union > 0 else 0.0 | |
| def match(predictions, truths, crowds, threshold): | |
| """Greedy matching at IoU 0.5; a hit on a crowd region is neither right nor wrong.""" | |
| used, tp, fp = set(), 0, 0 | |
| for score, box in sorted(predictions, key=lambda p: -p[0]): | |
| if score < threshold: | |
| continue | |
| best, best_iou = None, 0.5 | |
| for i, truth in enumerate(truths): | |
| if i not in used and iou(box, truth) >= best_iou: | |
| best, best_iou = i, iou(box, truth) | |
| if best is not None: | |
| used.add(best) | |
| tp += 1 | |
| elif not any(iou(box, crowd) >= 0.5 for crowd in crowds): | |
| fp += 1 | |
| return tp, fp, len(truths) - len(used) | |
| def load_pictures(data_dir: str, count: int): | |
| from PIL import Image | |
| with open(os.path.join(HERE, "coco60.json"), encoding="utf-8") as f: | |
| sample = json.load(f) | |
| folder = os.path.join(data_dir, "coco-val2017") | |
| os.makedirs(folder, exist_ok=True) | |
| items = sample["images"][:count] | |
| for item in items: | |
| path = os.path.join(folder, item["file"]) | |
| if not os.path.isfile(path): | |
| urllib.request.urlretrieve(item["url"], path) | |
| item["image"] = Image.open(path).convert("RGB") | |
| return items, sample["expected_f1"] | |
| def score(model, items) -> dict: | |
| """F1 at each threshold over the pictures: one image pass per picture, then a question per category.""" | |
| tally = {t: {"tp": 0, "fp": 0, "fn": 0, "absent_hit": 0} for t in THRESHOLDS} | |
| absent_prompts, seconds = 0, 0.0 | |
| for item in items: | |
| start = time.perf_counter() | |
| model.image(item["image"]) | |
| answers = {c: model.ask(item["image"], c, SCORE_FLOOR) for c in list(item["present"]) + item["absent"]} | |
| seconds += time.perf_counter() - start | |
| absent_prompts += len(item["absent"]) | |
| for t in THRESHOLDS: | |
| for concept, truths in item["present"].items(): | |
| tp, fp, fn = match(answers[concept], truths, item["crowds"].get(concept, []), t) | |
| tally[t]["tp"] += tp | |
| tally[t]["fp"] += fp | |
| tally[t]["fn"] += fn | |
| for concept in item["absent"]: | |
| hits = sum(s >= t for s, _ in answers[concept]) | |
| tally[t]["fp"] += hits | |
| tally[t]["absent_hit"] += bool(hits) | |
| out = {"pictures": len(items), "seconds_per_picture": round(seconds / max(len(items), 1), 2)} | |
| for t in THRESHOLDS: | |
| tp, fp, fn = tally[t]["tp"], tally[t]["fp"], tally[t]["fn"] | |
| precision = tp / (tp + fp) if tp + fp else 0.0 | |
| recall = tp / (tp + fn) if tp + fn else 0.0 | |
| out[str(t)] = { | |
| "precision": round(precision, 3), "recall": round(recall, 3), | |
| "f1": round(2 * precision * recall / (precision + recall), 3) if precision + recall else 0.0, | |
| "absent_prompts_answered": round(tally[t]["absent_hit"] / max(absent_prompts, 1), 3), | |
| } | |
| return out | |
| # --------------------------------------------------------------------------- the models | |
| class MiniModel: | |
| def __init__(self, bundle, provider, threads, image_graph=None): | |
| sys.path.insert(0, KIT) | |
| from sam3edge_mini import Mini | |
| self.m = Mini(bundle, threads=threads, provider=provider, force=True, image_graph=image_graph) | |
| self.masks = True | |
| self.where = {"image": f"{self.m.image_provider} ({self.m.image_graph})", "rest": provider} | |
| self.image_key = next(k for k, v in self.m.meta["files"].items() if v == self.m.image_graph and k.startswith("image")) | |
| self._features = None | |
| def image(self, picture): | |
| self._features = self.m.encode_image(picture) | |
| def forget_text(self): | |
| self.m._phrases.clear() | |
| def ask(self, picture, concept, threshold, masks=False): | |
| found = self.m.detect(picture, [concept], threshold=threshold, masks=masks, features=self._features) | |
| return [(d["score"], d["box_xyxy"]) for d in found] | |
| class MicroModel: | |
| def __init__(self, bundle, provider, threads, kit): | |
| sys.path.insert(0, kit) | |
| from sam3edge_micro import Micro | |
| self.m = Micro(bundle, threads=threads, provider=provider, force=True) | |
| self.masks = False | |
| self.where = {"image": getattr(self.m, "image_provider", provider), "rest": provider} | |
| self._level = None | |
| def image(self, picture): | |
| self._level = self.m.encode_image(picture) | |
| def forget_text(self): | |
| pass # the text was encoded when the bundle was made | |
| def ask(self, picture, concept, threshold, masks=False): | |
| level = self._level | |
| self.m.encode_image = lambda _picture: level # reuse the features for every concept | |
| try: | |
| found = self.m.detect(picture, [concept], threshold) | |
| finally: | |
| del self.m.encode_image | |
| return [(d["score"], d["box_xyxy"]) for d in found] | |
| class OtherModel: | |
| """Another publisher's ONNX export of the full SAM 3 in the common three-graph layout | |
| (onnx/vision_encoder*.onnx, text_encoder*.onnx, prompt_decoder*.onnx), scored the same way: | |
| the same pictures, the same 0.5 IoU suppression, its masks computed and ignored.""" | |
| def __init__(self, folder, suffix, provider, threads): | |
| import numpy as np | |
| import onnxruntime as ort | |
| from tokenizers import Tokenizer | |
| self.np = np | |
| options = ort.SessionOptions() | |
| options.log_severity_level = 3 | |
| if threads: | |
| options.intra_op_num_threads = threads | |
| gpu = ("CUDAExecutionProvider", {"arena_extend_strategy": "kSameAsRequested", | |
| "cudnn_conv_algo_search": "HEURISTIC"}) | |
| if provider == "cuda" and hasattr(ort, "preload_dlls"): | |
| try: | |
| ort.preload_dlls() | |
| except Exception: | |
| pass | |
| def load(name, where): | |
| path = os.path.join(folder, "onnx", f"{name}{suffix}.onnx") | |
| providers = [gpu, "CPUExecutionProvider"] if where == "cuda" else ["CPUExecutionProvider"] | |
| session = ort.InferenceSession(path, options, providers=providers) | |
| if where == "cuda" and "CUDAExecutionProvider" not in session.get_providers(): | |
| raise RuntimeError("ONNX Runtime did not load its CUDA provider") | |
| return session | |
| # the text encoder runs once per phrase: on the CPU, to leave the GPU's memory to the other two | |
| self.vision, self.text, self.decoder = load("vision_encoder", provider), load("text_encoder", "cpu"), load("prompt_decoder", provider) | |
| self.tokenizer = Tokenizer.from_file(os.path.join(folder, "tokenizer.json")) | |
| with open(os.path.join(folder, "tokenizer_config.json"), encoding="utf-8") as f: | |
| pad = json.load(f).get("pad_token", "<|endoftext|>") | |
| pad = pad["content"] if isinstance(pad, dict) else pad | |
| self.tokenizer.enable_padding(length=32, pad_id=self.tokenizer.token_to_id(pad), pad_token=pad) | |
| self.tokenizer.enable_truncation(32) | |
| self.masks = False | |
| self.where = {"image": provider, "rest": f"{provider} (text encoder on the cpu)"} | |
| self._phrases, self._fpn, self._size = {}, None, None | |
| self._outputs = [o.name for o in self.decoder.get_outputs() if o.name in ("pred_boxes", "pred_logits", "presence_logits")] | |
| def image(self, picture): | |
| np = self.np | |
| from PIL import Image | |
| pixels = np.asarray(picture.resize((1008, 1008), Image.BILINEAR), dtype=np.float32) / 255.0 | |
| pixels = ((pixels - 0.5) / 0.5).transpose(2, 0, 1)[None] | |
| self._fpn, self._size = self.vision.run(None, {"pixel_values": pixels})[:3], picture.size | |
| def forget_text(self): | |
| self._phrases.clear() | |
| def ask(self, picture, concept, threshold, masks=False): | |
| np = self.np | |
| if concept not in self._phrases: | |
| enc = self.tokenizer.encode(concept) | |
| ids = np.array([enc.ids], dtype=np.int64) | |
| mask = np.array([enc.attention_mask], dtype=np.int64) | |
| self._phrases[concept] = (self.text.run(None, {"input_ids": ids, "attention_mask": mask})[0], mask) | |
| features, mask = self._phrases[concept] | |
| out = dict(zip(self._outputs, self.decoder.run(self._outputs, { | |
| "fpn_hidden_state_0": self._fpn[0], "fpn_hidden_state_1": self._fpn[1], "fpn_hidden_state_2": self._fpn[2], | |
| "text_features": features, "attention_mask": mask}))) | |
| sigmoid = lambda x: 1.0 / (1.0 + np.exp(-x)) # noqa: E731 | |
| scores = sigmoid(out["pred_logits"].reshape(-1)) * float(sigmoid(out["presence_logits"].reshape(-1)[0])) | |
| width, height = self._size | |
| boxes = out["pred_boxes"].reshape(-1, 4) * np.array([width, height, width, height], dtype=np.float32) | |
| kept = [] | |
| for i in np.argsort(-scores): | |
| if scores[i] < threshold: | |
| break | |
| box = [float(v) for v in boxes[i]] | |
| if all(iou(box, other) < 0.5 for _, other in kept): | |
| kept.append((float(scores[i]), box)) | |
| return kept | |
| # --------------------------------------------------------------------------- one model, in its own process | |
| def peak_rss_mb(): | |
| try: | |
| import resource | |
| return round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024) | |
| except ImportError: | |
| try: | |
| import psutil | |
| info = psutil.Process().memory_info() | |
| return round(getattr(info, "peak_wset", info.rss) / 1e6) | |
| except Exception: | |
| return None | |
| def gpu_memory_mb(): | |
| """What nvidia-smi says this process holds on the GPU, or None.""" | |
| try: | |
| text = subprocess.run(["nvidia-smi", "--query-compute-apps=pid,used_memory", "--format=csv,noheader,nounits"], | |
| capture_output=True, text=True, timeout=20).stdout | |
| for line in text.splitlines(): | |
| pid, used = [part.strip() for part in line.split(",")] | |
| if int(pid) == os.getpid(): | |
| return int(used) | |
| except Exception: | |
| pass | |
| return None | |
| def build(spec: dict, provider: str, threads): | |
| if spec["kind"] == "mini": | |
| return MiniModel(spec["path"], provider, threads, spec.get("image_graph")) | |
| if spec["kind"] == "micro": | |
| return MicroModel(spec["path"], provider, threads, spec["kit"]) | |
| return OtherModel(spec["path"], spec["suffix"], provider, threads) | |
| def time_model(model, items, threshold, runs): | |
| """Medians over the timing pictures: the image pass, a first phrase, one more phrase, and masks.""" | |
| image_s, first_s, next_s, mask_s, answers = [], [], [], [], [] | |
| for item in items[:TIMING_PICTURES]: | |
| concepts = (list(item["present"]) + item["absent"])[:2] | |
| model.image(item["image"]) | |
| model.ask(item["image"], concepts[0], threshold) # warm-up | |
| for _ in range(runs): | |
| model.forget_text() | |
| t0 = time.perf_counter() | |
| model.image(item["image"]) | |
| t1 = time.perf_counter() | |
| first = model.ask(item["image"], concepts[0], threshold) | |
| t2 = time.perf_counter() | |
| more = model.ask(item["image"], concepts[1], threshold) | |
| t3 = time.perf_counter() | |
| image_s.append(t1 - t0), first_s.append(t2 - t0), next_s.append(t3 - t2) | |
| if model.masks: # the same phrase again, with and without masks: the difference is the mask head | |
| t4 = time.perf_counter() | |
| model.ask(item["image"], concepts[0], threshold, masks=True) | |
| t5 = time.perf_counter() | |
| model.ask(item["image"], concepts[0], threshold) | |
| mask_s.append((t5 - t4) - (time.perf_counter() - t5)) | |
| answers.append({concepts[0]: [round(s, 4) for s, _ in first], concepts[1]: [round(s, 4) for s, _ in more]}) | |
| out = {"image_s": round(statistics.median(image_s), 3), "first_phrase_s": round(statistics.median(first_s), 3), | |
| "next_phrase_s": round(statistics.median(next_s), 3)} | |
| if mask_s: | |
| out["masks_extra_s"] = round(max(statistics.median(mask_s), 0.0), 3) | |
| return out, answers | |
| def run_one(spec: dict, provider: str, args) -> dict: | |
| items, _ = load_pictures(args.data, max(args.coco, TIMING_PICTURES)) | |
| start = time.perf_counter() | |
| model = build(spec, provider, args.threads) | |
| result = {"name": spec["name"], "provider": provider, "runs_on": model.where, | |
| "image_graph": getattr(model, "image_key", "image"), | |
| "load_s": round(time.perf_counter() - start, 2), "threshold": spec["threshold"]} | |
| timing, answers = time_model(model, items, spec["threshold"], args.runs) | |
| result.update(timing) | |
| result["timing_answers"] = answers | |
| if args.coco and not spec.get("timing_only"): | |
| result["coco"] = score(model, items[: args.coco]) | |
| result["peak_memory_mb"] = peak_rss_mb() | |
| if provider == "cuda": | |
| result["gpu_memory_mb"] = gpu_memory_mb() | |
| return result | |
| # --------------------------------------------------------------------------- the whole run | |
| def environment(provider: str) -> dict: | |
| import onnxruntime as ort | |
| env = {"machine": platform.node(), "system": f"{platform.system()} {platform.release()}", | |
| "python": platform.python_version(), "onnxruntime": ort.__version__, | |
| "onnxruntime_providers": ort.get_available_providers(), "cpu_count": os.cpu_count(), "provider": provider} | |
| try: | |
| with open("/proc/cpuinfo") as f: | |
| env["cpu"] = next(line.split(":", 1)[1].strip() for line in f if line.startswith("model name")) | |
| with open("/proc/meminfo") as f: | |
| env["memory_gb"] = round(int(next(line for line in f if line.startswith("MemTotal")).split()[1]) / 1e6, 1) | |
| except Exception: | |
| env["cpu"] = platform.processor() | |
| try: | |
| env["gpu"] = subprocess.run( | |
| ["nvidia-smi", "--query-gpu=name,driver_version,memory.total", "--format=csv,noheader"], | |
| capture_output=True, text=True, timeout=20).stdout.strip() | |
| except Exception: | |
| env["gpu"] = None | |
| return env | |
| OFFLINE = False | |
| def fetch(repo: str, patterns, target: str, skip=None) -> str: | |
| """Download what is missing of ``patterns`` into ``target``; with --offline, use what is already there.""" | |
| if OFFLINE: | |
| return target | |
| from huggingface_hub import snapshot_download | |
| return snapshot_download(repo, allow_patterns=patterns, ignore_patterns=skip, local_dir=target) | |
| def child(spec: dict, provider: str, args) -> dict: | |
| command = [sys.executable, os.path.abspath(__file__), "--one", json.dumps(spec), "--provider", provider, | |
| "--data", args.data, "--coco", str(args.coco), "--runs", str(args.runs)] | |
| if args.threads: | |
| command += ["--threads", str(args.threads)] | |
| done = subprocess.run(command, capture_output=True, text=True) | |
| line = next((ln for ln in done.stdout.splitlines() if ln.startswith("RESULT ")), None) | |
| if line is None: | |
| tail = (done.stderr or done.stdout).strip().splitlines()[-3:] | |
| return {"name": spec["name"], "provider": provider, "error": " | ".join(tail)[-600:]} | |
| return json.loads(line[7:]) | |
| def table(rows, expected) -> str: | |
| lines = ["| Model | Runs on | Image pass | First phrase | Next phrase | Masks, extra | Memory | GPU memory | F1 at 0.5 | F1 at 0.3 | Expected F1 |", | |
| "|---|---|---|---|---|---|---|---|---|---|---|"] | |
| for r in rows: | |
| if "error" in r: | |
| lines.append(f"| {r['name']} | {r['provider']} | failed: {r['error'][:120]} | | | | | | | | |") | |
| continue | |
| coco = r.get("coco", {}) | |
| want = expected.get(r["name"] + (" float" if r.get("image_graph") == "image_fp32" else ""), {}) | |
| f1 = lambda t: f"{coco[t]['f1']:.3f}" if t in coco else "" # noqa: E731 | |
| lines.append( | |
| f"| {r['name']} | {r['provider']} | {r['image_s']} s | {r['first_phrase_s']} s | {r['next_phrase_s']} s | " | |
| f"{str(r['masks_extra_s']) + ' s' if 'masks_extra_s' in r else ''} | {r.get('peak_memory_mb') or ''} MB | " | |
| f"{str(r['gpu_memory_mb']) + ' MB' if r.get('gpu_memory_mb') else ''} | {f1('0.5')} | {f1('0.3')} | " | |
| f"{want.get('0.5', '')} / {want.get('0.3', '')} |" | |
| ) | |
| return "\n".join(lines) | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) | |
| parser.add_argument("--provider", choices=["cpu", "cuda"], default="cpu") | |
| parser.add_argument("--bundles", default=",".join(MINI_BUNDLES), help="sam3edge-mini bundles to run; 'none' for none") | |
| parser.add_argument("--micro", default="", help="sam3edge-micro bundles to run too, e.g. fast-672,quality-336") | |
| parser.add_argument("--other", default="", help="another publisher's three-graph ONNX export, e.g. tepirale/sam3-ONNX") | |
| parser.add_argument("--other-suffix", default=None, help="its precision: '' (fp32), _fp16, _uint8 ... " | |
| "(default: fp32 on a GPU, _uint8 on a CPU)") | |
| parser.add_argument("--coco", type=int, default=60, help="how many of the 60 pictures to score (0 skips accuracy)") | |
| parser.add_argument("--runs", type=int, default=3) | |
| parser.add_argument("--threads", type=int) | |
| parser.add_argument("--no-cpu-check", action="store_true", help="with cuda: skip the CPU run each GPU run is compared with") | |
| parser.add_argument("--offline", action="store_true", help="download no models: use the bundles already in this folder") | |
| parser.add_argument("--image-graph", choices=["image", "image_fp32"], | |
| help="which form of a bundle's image encoder to run (default: the one that suits the provider)") | |
| parser.add_argument("--data", default=os.path.join(KIT, "data")) | |
| parser.add_argument("--out", default=os.path.join(KIT, "results")) | |
| parser.add_argument("--one", help=argparse.SUPPRESS) | |
| args = parser.parse_args() | |
| if args.one: | |
| print("RESULT " + json.dumps(run_one(json.loads(args.one), args.provider, args)), flush=True) | |
| return 0 | |
| global OFFLINE | |
| OFFLINE = args.offline | |
| os.makedirs(args.out, exist_ok=True) | |
| os.makedirs(args.data, exist_ok=True) | |
| env = environment(args.provider) | |
| print(json.dumps(env, indent=1), flush=True) | |
| if args.provider == "cuda" and "CUDAExecutionProvider" not in env["onnxruntime_providers"]: | |
| print('This onnxruntime has no CUDA provider. Install the GPU build: pip install "onnxruntime-gpu[cuda,cudnn]"') | |
| return 2 | |
| items, expected = load_pictures(args.data, max(args.coco, TIMING_PICTURES)) | |
| print(f"{len(items)} pictures ready", flush=True) | |
| specs = [] | |
| for name in [b for b in args.bundles.split(",") if b and b != "none"]: | |
| # the float image graph (1.8 GB for SAM 3's own encoder) is only of use to a GPU | |
| wants_float = args.provider == "cuda" or args.image_graph == "image_fp32" | |
| folder = fetch(MINI_REPO, [f"bundles/{name}/*"], KIT, None if wants_float else ["*image-fp32*"]) | |
| with open(os.path.join(folder, "bundles", name, "mini.json"), encoding="utf-8") as f: | |
| threshold = json.load(f)["threshold"] | |
| specs.append({"kind": "mini", "name": f"sam3edge-mini/{name}", "path": os.path.join(folder, "bundles", name), | |
| "threshold": threshold, "image_graph": args.image_graph}) | |
| if args.micro: | |
| kit = fetch(MICRO_REPO, ["sam3edge_micro/*"] + [f"bundles/{n}/*" for n in args.micro.split(",")], | |
| os.path.join(KIT, "micro")) | |
| for name in args.micro.split(","): | |
| with open(os.path.join(kit, "bundles", name, "micro.json"), encoding="utf-8") as f: | |
| threshold = json.load(f)["threshold"] | |
| specs.append({"kind": "micro", "name": f"sam3edge-micro/{name}", "path": os.path.join(kit, "bundles", name), | |
| "kit": kit, "threshold": threshold}) | |
| if args.other: | |
| suffix = args.other_suffix if args.other_suffix is not None else ("" if args.provider == "cuda" else "_uint8") | |
| patterns = [f"onnx/{g}{suffix}.onnx*" for g in ("vision_encoder", "text_encoder", "prompt_decoder")] | |
| folder = fetch(args.other, patterns + ["*.json", "*.txt"], os.path.join(KIT, "other", args.other.replace("/", "__"))) | |
| specs.append({"kind": "other", "name": f"{args.other} {suffix.strip('_') or 'fp32'} (full SAM 3, 1008 px)", | |
| "path": folder, "suffix": suffix, "threshold": 0.5}) | |
| rows = [] | |
| for spec in specs: | |
| print(f"running {spec['name']} on {args.provider} ...", flush=True) | |
| row = child(spec, args.provider, args) | |
| if args.provider == "cuda" and not args.no_cpu_check and "error" not in row and spec["kind"] != "other": | |
| print(" and on the cpu, to compare the answers ...", flush=True) | |
| # the same form of the image encoder as the GPU ran, so that only the processor differs | |
| cpu = child({**spec, "timing_only": True, "image_graph": row.get("image_graph")}, "cpu", args) | |
| if "error" not in cpu: | |
| gaps = [abs(a - b) for g, c in zip(row["timing_answers"], cpu["timing_answers"]) | |
| for key in g for a, b in zip(g[key], c[key])] | |
| row["cpu_check"] = { | |
| "same_counts": all(len(g[k]) == len(c[k]) for g, c in zip(row["timing_answers"], cpu["timing_answers"]) for k in g), | |
| "largest_score_gap": round(max(gaps), 4) if gaps else 0.0, | |
| "cpu_first_phrase_s": cpu["first_phrase_s"], "cpu_image_s": cpu["image_s"], | |
| } | |
| rows.append(row) | |
| print(" " + json.dumps({k: v for k, v in row.items() if k != "timing_answers"}), flush=True) | |
| stem = os.path.join(args.out, f"{platform.node() or 'machine'}-{args.provider}") | |
| with open(stem + ".json", "w", encoding="utf-8") as f: | |
| json.dump({"environment": env, "expected_f1": expected, "rows": rows}, f, indent=1) | |
| text = table(rows, expected) | |
| with open(stem + ".md", "w", encoding="utf-8") as f: | |
| f.write(f"Machine: {env.get('cpu')}; GPU: {env.get('gpu')}; onnxruntime {env['onnxruntime']}; provider {args.provider}\n\n{text}\n") | |
| print("\n" + text + f"\n\nwritten: {stem}.json and {stem}.md") | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |