faxenoff's picture
cloud build tools (matrix trt11.3, 5 archs); the tools bucket is read-only since 10-07
40da26e verified
Raw History Blame Contribute Delete
9.63 kB
#!/usr/bin/env python3
"""Build every TRT engine this VM's GPU can produce, straight from build_matrix.json.
`build_engines.{ps1,sh}` builds ONE model per invocation from environment
variables. This driver reads the generated matrix and makes those invocations —
five models, the right buckets, the right per-arch precision — so the operator
types a GPU arch's worth of work as one command and cannot mistype a precision.
One driver for both operating systems on purpose. The Windows and Linux lanes
differ only in which wrapper they exec; duplicating the matrix walk in PowerShell
and bash is how the two halves of a {win_x64, linux_x64} pair start disagreeing
about what "the same engine" means.
python run_matrix.py # everything the matrix lists
python run_matrix.py --only code-daemon-embed-v1
python run_matrix.py --skip ultracode-ner
python run_matrix.py --dry-run # print the invocations, build nothing
Environment it passes through untouched (set them before calling):
HF_TOKEN write token — required unless DEST_PREFIX is set
HF_UPLOAD_REPO override the destination repo (default: the source repo)
DEST_PREFIX S3 upload root, if you also want a bucket copy
TOOLS_URL where build_engines + the .py workers are hosted
WORKDIR scratch dir (default: ~/uc-build)
Exit code is the number of FAILED models, so a wrapper can branch on it. A model
that fails does not stop the rest: on a rented box the expensive resource is the
hour, and one bad model should not cost the other four.
"""
import argparse
import json
import os
import platform
import subprocess
import sys
import time
import urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
def log(msg):
print(msg, flush=True)
def detect_sm():
"""`sm_86` from nvidia-smi's compute_cap. The engine name depends on it, so a
guess is not acceptable - no GPU means no build, not a default."""
try:
out = subprocess.check_output(
["nvidia-smi", "--query-gpu=compute_cap,name", "--format=csv,noheader"],
stderr=subprocess.STDOUT, universal_newlines=True)
except Exception as e:
raise SystemExit("nvidia-smi failed (%s) - no NVIDIA driver on this VM?" % e)
first = out.strip().splitlines()[0]
cap, _, name = first.partition(",")
return "sm_" + cap.strip().replace(".", ""), name.strip()
def host_os_arch():
return "win_x64" if platform.system() == "Windows" else "linux_x64"
def load_matrix(path_or_url):
if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
with urllib.request.urlopen(path_or_url, timeout=30) as r:
return json.loads(r.read().decode("utf-8"))
with open(path_or_url, encoding="utf-8") as f:
return json.load(f)
def resolve_matrix_path(explicit, tools_url):
if explicit:
return explicit
beside = os.path.join(HERE, "build_matrix.json")
if os.path.isfile(beside):
return beside
if tools_url:
return tools_url.rstrip("/") + "/build_matrix.json"
raise SystemExit(
"build_matrix.json not found beside this script and TOOLS_URL is unset - "
"pass --matrix <path|url>")
def wrapper_cmd(os_arch):
"""The per-OS wrapper, taken from beside this file (staged together)."""
if os_arch == "win_x64":
script = os.path.join(HERE, "build_engines.ps1")
if not os.path.isfile(script):
raise SystemExit("build_engines.ps1 not found beside run_matrix.py")
return ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", script]
script = os.path.join(HERE, "build_engines.sh")
if not os.path.isfile(script):
raise SystemExit("build_engines.sh not found beside run_matrix.py")
return ["bash", script]
def env_for(entry, arch, base_env):
"""Translate one matrix row into the wrapper's environment contract."""
cell = entry["trt"][arch]
env = dict(base_env)
env["MODEL_ID"] = entry["model_id"]
env["ROLE"] = entry["role"]
# "__single__", not "": Windows does not store empty environment variables -
# SetEnvironmentVariable(name, "") DELETES the variable, so the wrapper sees
# "unset" and falls back to its "s m l" default. That built three bucketed
# engines for a single-profile model, under names the daemon never looks up.
# Both wrappers already treat __single__ as "one engine, no bucket suffix".
env["BUCKETS"] = " ".join(entry["buckets"]) if entry["buckets"] else "__single__"
env["PRECISION"] = cell["precision"]
env["HF_MODEL_REPO"] = entry["hf_repo"]
# Destination defaults to the source repo - the daemon reads engines from the
# same place it reads the ONNX. An explicit HF_UPLOAD_REPO still wins.
env.setdefault("HF_UPLOAD_REPO", entry["hf_repo"])
if base_env.get("HF_UPLOAD_REPO"):
env["HF_UPLOAD_REPO"] = base_env["HF_UPLOAD_REPO"]
return env
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--matrix", default=None, help="path or URL to build_matrix.json")
ap.add_argument("--only", action="append", default=[], help="build only this model id (repeatable)")
ap.add_argument("--skip", action="append", default=[], help="skip this model id (repeatable)")
ap.add_argument("--arch", default=None, help="override the detected sm_XX (testing only)")
ap.add_argument("--dry-run", action="store_true", help="print invocations, build nothing")
args = ap.parse_args()
tools_url = os.environ.get("TOOLS_URL", "")
matrix = load_matrix(resolve_matrix_path(args.matrix, tools_url))
os_arch = host_os_arch()
if args.arch:
sm, gpu = args.arch, "(overridden)"
else:
sm, gpu = detect_sm()
log("=" * 70)
log(" ultracode cloud engine matrix")
log(" GPU : %s -> %s" % (gpu, sm))
log(" platform : %s" % os_arch)
log(" TRT tag : %s (engine names carry it; a mismatch is unloadable)" % matrix["trt_tag"])
log("=" * 70)
if sm not in matrix["cuda_archs"]:
log("")
log(" !! %s is not in the cloud matrix %s." % (sm, matrix["cuda_archs"]))
if sm == "sm_120":
log(" sm_120 is the LOCAL lane - it is built at a desk and already published.")
else:
log(" Add it to build_config.cuda_archs and regenerate, or rent a different card.")
return 1
base_env = dict(os.environ)
# The wrapper installs TensorRT before the first build; without the exact
# version it would either guess or let build_trt_engine.py fail on "cannot
# determine the TensorRT version".
if matrix.get("trt_pip_version"):
base_env["TRT_PIP_VERSION"] = matrix["trt_pip_version"]
if not base_env.get("HF_TOKEN") and not base_env.get("DEST_PREFIX"):
log("")
log(" !! neither HF_TOKEN nor DEST_PREFIX is set - the engines would be built and discarded.")
return 1
todo = []
for entry in matrix["models"]:
mid = entry["model_id"]
if args.only and mid not in args.only:
continue
if mid in args.skip:
log(" [skip] %s (--skip)" % mid)
continue
# A private HF repo cannot serve the ONNX to a machine holding only a
# write token for the OTHER repos. Say so once, here, instead of letting
# the wrapper fail on a 401 five minutes in.
if entry.get("hf_public") is False and not base_env.get("SRC_ONNX_S3"):
log(" [skip] %s - HF repo %s is not public and no SRC_ONNX_S3 given"
% (mid, entry["hf_repo"]))
continue
todo.append(entry)
if not todo:
log(" nothing to build")
return 0
cmd = wrapper_cmd(os_arch)
results = []
for entry in todo:
cell = entry["trt"][sm]
n = max(1, len(entry["buckets"]))
log("")
log("-" * 70)
log(" %s [%s] %d engine(s) buckets=%s"
% (entry["model_id"], cell["precision"],
n, ",".join(entry["buckets"]) or "(single)"))
log(" source: %s :: %s" % (entry["hf_repo"], cell["source"]["file"]))
if cell["source"]["ptq_on_vm"]:
log(" note : PTQ runs ON THIS VM - the calibration corpus must be staged")
log("-" * 70)
env = env_for(entry, sm, base_env)
if args.dry_run:
keys = ("MODEL_ID", "ROLE", "BUCKETS", "PRECISION", "HF_MODEL_REPO", "HF_UPLOAD_REPO")
for k in keys:
log(" %s=%s" % (k, env.get(k, "")))
results.append((entry["model_id"], "dry-run", 0.0))
continue
t0 = time.time()
rc = subprocess.call(cmd, env=env)
dt = time.time() - t0
results.append((entry["model_id"], "ok" if rc == 0 else "FAILED rc=%d" % rc, dt))
if rc != 0:
log(" !! %s failed (rc=%d) - continuing with the rest" % (entry["model_id"], rc))
log("")
log("=" * 70)
log(" SUMMARY %s / %s" % (sm, os_arch))
failed = 0
for mid, status, dt in results:
mark = "OK " if status in ("ok", "dry-run") else "FAIL"
if status.startswith("FAILED"):
failed += 1
log(" [%s] %-28s %-14s %6.1fs" % (mark, mid, status, dt))
log("=" * 70)
if failed:
log(" %d model(s) FAILED - engines for those cells are missing." % failed)
return failed
if __name__ == "__main__":
raise SystemExit(main())