File size: 9,627 Bytes
40da26e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 | #!/usr/bin/env python3
"""Build every TRT engine this VM's GPU can produce, straight from build_matrix.json.
`build_engines.{ps1,sh}` builds ONE model per invocation from environment
variables. This driver reads the generated matrix and makes those invocations —
five models, the right buckets, the right per-arch precision — so the operator
types a GPU arch's worth of work as one command and cannot mistype a precision.
One driver for both operating systems on purpose. The Windows and Linux lanes
differ only in which wrapper they exec; duplicating the matrix walk in PowerShell
and bash is how the two halves of a {win_x64, linux_x64} pair start disagreeing
about what "the same engine" means.
python run_matrix.py # everything the matrix lists
python run_matrix.py --only code-daemon-embed-v1
python run_matrix.py --skip ultracode-ner
python run_matrix.py --dry-run # print the invocations, build nothing
Environment it passes through untouched (set them before calling):
HF_TOKEN write token — required unless DEST_PREFIX is set
HF_UPLOAD_REPO override the destination repo (default: the source repo)
DEST_PREFIX S3 upload root, if you also want a bucket copy
TOOLS_URL where build_engines + the .py workers are hosted
WORKDIR scratch dir (default: ~/uc-build)
Exit code is the number of FAILED models, so a wrapper can branch on it. A model
that fails does not stop the rest: on a rented box the expensive resource is the
hour, and one bad model should not cost the other four.
"""
import argparse
import json
import os
import platform
import subprocess
import sys
import time
import urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
def log(msg):
print(msg, flush=True)
def detect_sm():
"""`sm_86` from nvidia-smi's compute_cap. The engine name depends on it, so a
guess is not acceptable - no GPU means no build, not a default."""
try:
out = subprocess.check_output(
["nvidia-smi", "--query-gpu=compute_cap,name", "--format=csv,noheader"],
stderr=subprocess.STDOUT, universal_newlines=True)
except Exception as e:
raise SystemExit("nvidia-smi failed (%s) - no NVIDIA driver on this VM?" % e)
first = out.strip().splitlines()[0]
cap, _, name = first.partition(",")
return "sm_" + cap.strip().replace(".", ""), name.strip()
def host_os_arch():
return "win_x64" if platform.system() == "Windows" else "linux_x64"
def load_matrix(path_or_url):
if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
with urllib.request.urlopen(path_or_url, timeout=30) as r:
return json.loads(r.read().decode("utf-8"))
with open(path_or_url, encoding="utf-8") as f:
return json.load(f)
def resolve_matrix_path(explicit, tools_url):
if explicit:
return explicit
beside = os.path.join(HERE, "build_matrix.json")
if os.path.isfile(beside):
return beside
if tools_url:
return tools_url.rstrip("/") + "/build_matrix.json"
raise SystemExit(
"build_matrix.json not found beside this script and TOOLS_URL is unset - "
"pass --matrix <path|url>")
def wrapper_cmd(os_arch):
"""The per-OS wrapper, taken from beside this file (staged together)."""
if os_arch == "win_x64":
script = os.path.join(HERE, "build_engines.ps1")
if not os.path.isfile(script):
raise SystemExit("build_engines.ps1 not found beside run_matrix.py")
return ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", script]
script = os.path.join(HERE, "build_engines.sh")
if not os.path.isfile(script):
raise SystemExit("build_engines.sh not found beside run_matrix.py")
return ["bash", script]
def env_for(entry, arch, base_env):
"""Translate one matrix row into the wrapper's environment contract."""
cell = entry["trt"][arch]
env = dict(base_env)
env["MODEL_ID"] = entry["model_id"]
env["ROLE"] = entry["role"]
# "__single__", not "": Windows does not store empty environment variables -
# SetEnvironmentVariable(name, "") DELETES the variable, so the wrapper sees
# "unset" and falls back to its "s m l" default. That built three bucketed
# engines for a single-profile model, under names the daemon never looks up.
# Both wrappers already treat __single__ as "one engine, no bucket suffix".
env["BUCKETS"] = " ".join(entry["buckets"]) if entry["buckets"] else "__single__"
env["PRECISION"] = cell["precision"]
env["HF_MODEL_REPO"] = entry["hf_repo"]
# Destination defaults to the source repo - the daemon reads engines from the
# same place it reads the ONNX. An explicit HF_UPLOAD_REPO still wins.
env.setdefault("HF_UPLOAD_REPO", entry["hf_repo"])
if base_env.get("HF_UPLOAD_REPO"):
env["HF_UPLOAD_REPO"] = base_env["HF_UPLOAD_REPO"]
return env
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--matrix", default=None, help="path or URL to build_matrix.json")
ap.add_argument("--only", action="append", default=[], help="build only this model id (repeatable)")
ap.add_argument("--skip", action="append", default=[], help="skip this model id (repeatable)")
ap.add_argument("--arch", default=None, help="override the detected sm_XX (testing only)")
ap.add_argument("--dry-run", action="store_true", help="print invocations, build nothing")
args = ap.parse_args()
tools_url = os.environ.get("TOOLS_URL", "")
matrix = load_matrix(resolve_matrix_path(args.matrix, tools_url))
os_arch = host_os_arch()
if args.arch:
sm, gpu = args.arch, "(overridden)"
else:
sm, gpu = detect_sm()
log("=" * 70)
log(" ultracode cloud engine matrix")
log(" GPU : %s -> %s" % (gpu, sm))
log(" platform : %s" % os_arch)
log(" TRT tag : %s (engine names carry it; a mismatch is unloadable)" % matrix["trt_tag"])
log("=" * 70)
if sm not in matrix["cuda_archs"]:
log("")
log(" !! %s is not in the cloud matrix %s." % (sm, matrix["cuda_archs"]))
if sm == "sm_120":
log(" sm_120 is the LOCAL lane - it is built at a desk and already published.")
else:
log(" Add it to build_config.cuda_archs and regenerate, or rent a different card.")
return 1
base_env = dict(os.environ)
# The wrapper installs TensorRT before the first build; without the exact
# version it would either guess or let build_trt_engine.py fail on "cannot
# determine the TensorRT version".
if matrix.get("trt_pip_version"):
base_env["TRT_PIP_VERSION"] = matrix["trt_pip_version"]
if not base_env.get("HF_TOKEN") and not base_env.get("DEST_PREFIX"):
log("")
log(" !! neither HF_TOKEN nor DEST_PREFIX is set - the engines would be built and discarded.")
return 1
todo = []
for entry in matrix["models"]:
mid = entry["model_id"]
if args.only and mid not in args.only:
continue
if mid in args.skip:
log(" [skip] %s (--skip)" % mid)
continue
# A private HF repo cannot serve the ONNX to a machine holding only a
# write token for the OTHER repos. Say so once, here, instead of letting
# the wrapper fail on a 401 five minutes in.
if entry.get("hf_public") is False and not base_env.get("SRC_ONNX_S3"):
log(" [skip] %s - HF repo %s is not public and no SRC_ONNX_S3 given"
% (mid, entry["hf_repo"]))
continue
todo.append(entry)
if not todo:
log(" nothing to build")
return 0
cmd = wrapper_cmd(os_arch)
results = []
for entry in todo:
cell = entry["trt"][sm]
n = max(1, len(entry["buckets"]))
log("")
log("-" * 70)
log(" %s [%s] %d engine(s) buckets=%s"
% (entry["model_id"], cell["precision"],
n, ",".join(entry["buckets"]) or "(single)"))
log(" source: %s :: %s" % (entry["hf_repo"], cell["source"]["file"]))
if cell["source"]["ptq_on_vm"]:
log(" note : PTQ runs ON THIS VM - the calibration corpus must be staged")
log("-" * 70)
env = env_for(entry, sm, base_env)
if args.dry_run:
keys = ("MODEL_ID", "ROLE", "BUCKETS", "PRECISION", "HF_MODEL_REPO", "HF_UPLOAD_REPO")
for k in keys:
log(" %s=%s" % (k, env.get(k, "")))
results.append((entry["model_id"], "dry-run", 0.0))
continue
t0 = time.time()
rc = subprocess.call(cmd, env=env)
dt = time.time() - t0
results.append((entry["model_id"], "ok" if rc == 0 else "FAILED rc=%d" % rc, dt))
if rc != 0:
log(" !! %s failed (rc=%d) - continuing with the rest" % (entry["model_id"], rc))
log("")
log("=" * 70)
log(" SUMMARY %s / %s" % (sm, os_arch))
failed = 0
for mid, status, dt in results:
mark = "OK " if status in ("ok", "dry-run") else "FAIL"
if status.startswith("FAILED"):
failed += 1
log(" [%s] %-28s %-14s %6.1fs" % (mark, mid, status, dt))
log("=" * 70)
if failed:
log(" %d model(s) FAILED - engines for those cells are missing." % failed)
return failed
if __name__ == "__main__":
raise SystemExit(main())
|