#!/usr/bin/env python3 """Build every TRT engine this VM's GPU can produce, straight from build_matrix.json. `build_engines.{ps1,sh}` builds ONE model per invocation from environment variables. This driver reads the generated matrix and makes those invocations — five models, the right buckets, the right per-arch precision — so the operator types a GPU arch's worth of work as one command and cannot mistype a precision. One driver for both operating systems on purpose. The Windows and Linux lanes differ only in which wrapper they exec; duplicating the matrix walk in PowerShell and bash is how the two halves of a {win_x64, linux_x64} pair start disagreeing about what "the same engine" means. python run_matrix.py # everything the matrix lists python run_matrix.py --only code-daemon-embed-v1 python run_matrix.py --skip ultracode-ner python run_matrix.py --dry-run # print the invocations, build nothing Environment it passes through untouched (set them before calling): HF_TOKEN write token — required unless DEST_PREFIX is set HF_UPLOAD_REPO override the destination repo (default: the source repo) DEST_PREFIX S3 upload root, if you also want a bucket copy TOOLS_URL where build_engines + the .py workers are hosted WORKDIR scratch dir (default: ~/uc-build) Exit code is the number of FAILED models, so a wrapper can branch on it. A model that fails does not stop the rest: on a rented box the expensive resource is the hour, and one bad model should not cost the other four. """ import argparse import json import os import platform import subprocess import sys import time import urllib.request HERE = os.path.dirname(os.path.abspath(__file__)) def log(msg): print(msg, flush=True) def detect_sm(): """`sm_86` from nvidia-smi's compute_cap. The engine name depends on it, so a guess is not acceptable - no GPU means no build, not a default.""" try: out = subprocess.check_output( ["nvidia-smi", "--query-gpu=compute_cap,name", "--format=csv,noheader"], stderr=subprocess.STDOUT, universal_newlines=True) except Exception as e: raise SystemExit("nvidia-smi failed (%s) - no NVIDIA driver on this VM?" % e) first = out.strip().splitlines()[0] cap, _, name = first.partition(",") return "sm_" + cap.strip().replace(".", ""), name.strip() def host_os_arch(): return "win_x64" if platform.system() == "Windows" else "linux_x64" def load_matrix(path_or_url): if path_or_url.startswith("http://") or path_or_url.startswith("https://"): with urllib.request.urlopen(path_or_url, timeout=30) as r: return json.loads(r.read().decode("utf-8")) with open(path_or_url, encoding="utf-8") as f: return json.load(f) def resolve_matrix_path(explicit, tools_url): if explicit: return explicit beside = os.path.join(HERE, "build_matrix.json") if os.path.isfile(beside): return beside if tools_url: return tools_url.rstrip("/") + "/build_matrix.json" raise SystemExit( "build_matrix.json not found beside this script and TOOLS_URL is unset - " "pass --matrix ") def wrapper_cmd(os_arch): """The per-OS wrapper, taken from beside this file (staged together).""" if os_arch == "win_x64": script = os.path.join(HERE, "build_engines.ps1") if not os.path.isfile(script): raise SystemExit("build_engines.ps1 not found beside run_matrix.py") return ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", script] script = os.path.join(HERE, "build_engines.sh") if not os.path.isfile(script): raise SystemExit("build_engines.sh not found beside run_matrix.py") return ["bash", script] def env_for(entry, arch, base_env): """Translate one matrix row into the wrapper's environment contract.""" cell = entry["trt"][arch] env = dict(base_env) env["MODEL_ID"] = entry["model_id"] env["ROLE"] = entry["role"] # "__single__", not "": Windows does not store empty environment variables - # SetEnvironmentVariable(name, "") DELETES the variable, so the wrapper sees # "unset" and falls back to its "s m l" default. That built three bucketed # engines for a single-profile model, under names the daemon never looks up. # Both wrappers already treat __single__ as "one engine, no bucket suffix". env["BUCKETS"] = " ".join(entry["buckets"]) if entry["buckets"] else "__single__" env["PRECISION"] = cell["precision"] env["HF_MODEL_REPO"] = entry["hf_repo"] # Destination defaults to the source repo - the daemon reads engines from the # same place it reads the ONNX. An explicit HF_UPLOAD_REPO still wins. env.setdefault("HF_UPLOAD_REPO", entry["hf_repo"]) if base_env.get("HF_UPLOAD_REPO"): env["HF_UPLOAD_REPO"] = base_env["HF_UPLOAD_REPO"] return env def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--matrix", default=None, help="path or URL to build_matrix.json") ap.add_argument("--only", action="append", default=[], help="build only this model id (repeatable)") ap.add_argument("--skip", action="append", default=[], help="skip this model id (repeatable)") ap.add_argument("--arch", default=None, help="override the detected sm_XX (testing only)") ap.add_argument("--dry-run", action="store_true", help="print invocations, build nothing") args = ap.parse_args() tools_url = os.environ.get("TOOLS_URL", "") matrix = load_matrix(resolve_matrix_path(args.matrix, tools_url)) os_arch = host_os_arch() if args.arch: sm, gpu = args.arch, "(overridden)" else: sm, gpu = detect_sm() log("=" * 70) log(" ultracode cloud engine matrix") log(" GPU : %s -> %s" % (gpu, sm)) log(" platform : %s" % os_arch) log(" TRT tag : %s (engine names carry it; a mismatch is unloadable)" % matrix["trt_tag"]) log("=" * 70) if sm not in matrix["cuda_archs"]: log("") log(" !! %s is not in the cloud matrix %s." % (sm, matrix["cuda_archs"])) if sm == "sm_120": log(" sm_120 is the LOCAL lane - it is built at a desk and already published.") else: log(" Add it to build_config.cuda_archs and regenerate, or rent a different card.") return 1 base_env = dict(os.environ) # The wrapper installs TensorRT before the first build; without the exact # version it would either guess or let build_trt_engine.py fail on "cannot # determine the TensorRT version". if matrix.get("trt_pip_version"): base_env["TRT_PIP_VERSION"] = matrix["trt_pip_version"] if not base_env.get("HF_TOKEN") and not base_env.get("DEST_PREFIX"): log("") log(" !! neither HF_TOKEN nor DEST_PREFIX is set - the engines would be built and discarded.") return 1 todo = [] for entry in matrix["models"]: mid = entry["model_id"] if args.only and mid not in args.only: continue if mid in args.skip: log(" [skip] %s (--skip)" % mid) continue # A private HF repo cannot serve the ONNX to a machine holding only a # write token for the OTHER repos. Say so once, here, instead of letting # the wrapper fail on a 401 five minutes in. if entry.get("hf_public") is False and not base_env.get("SRC_ONNX_S3"): log(" [skip] %s - HF repo %s is not public and no SRC_ONNX_S3 given" % (mid, entry["hf_repo"])) continue todo.append(entry) if not todo: log(" nothing to build") return 0 cmd = wrapper_cmd(os_arch) results = [] for entry in todo: cell = entry["trt"][sm] n = max(1, len(entry["buckets"])) log("") log("-" * 70) log(" %s [%s] %d engine(s) buckets=%s" % (entry["model_id"], cell["precision"], n, ",".join(entry["buckets"]) or "(single)")) log(" source: %s :: %s" % (entry["hf_repo"], cell["source"]["file"])) if cell["source"]["ptq_on_vm"]: log(" note : PTQ runs ON THIS VM - the calibration corpus must be staged") log("-" * 70) env = env_for(entry, sm, base_env) if args.dry_run: keys = ("MODEL_ID", "ROLE", "BUCKETS", "PRECISION", "HF_MODEL_REPO", "HF_UPLOAD_REPO") for k in keys: log(" %s=%s" % (k, env.get(k, ""))) results.append((entry["model_id"], "dry-run", 0.0)) continue t0 = time.time() rc = subprocess.call(cmd, env=env) dt = time.time() - t0 results.append((entry["model_id"], "ok" if rc == 0 else "FAILED rc=%d" % rc, dt)) if rc != 0: log(" !! %s failed (rc=%d) - continuing with the rest" % (entry["model_id"], rc)) log("") log("=" * 70) log(" SUMMARY %s / %s" % (sm, os_arch)) failed = 0 for mid, status, dt in results: mark = "OK " if status in ("ok", "dry-run") else "FAIL" if status.startswith("FAILED"): failed += 1 log(" [%s] %-28s %-14s %6.1fs" % (mark, mid, status, dt)) log("=" * 70) if failed: log(" %d model(s) FAILED - engines for those cells are missing." % failed) return failed if __name__ == "__main__": raise SystemExit(main())