File size: 9,627 Bytes
40da26e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
#!/usr/bin/env python3
"""Build every TRT engine this VM's GPU can produce, straight from build_matrix.json.

`build_engines.{ps1,sh}` builds ONE model per invocation from environment
variables. This driver reads the generated matrix and makes those invocations —
five models, the right buckets, the right per-arch precision — so the operator
types a GPU arch's worth of work as one command and cannot mistype a precision.

One driver for both operating systems on purpose. The Windows and Linux lanes
differ only in which wrapper they exec; duplicating the matrix walk in PowerShell
and bash is how the two halves of a {win_x64, linux_x64} pair start disagreeing
about what "the same engine" means.

    python run_matrix.py                       # everything the matrix lists
    python run_matrix.py --only code-daemon-embed-v1
    python run_matrix.py --skip ultracode-ner
    python run_matrix.py --dry-run             # print the invocations, build nothing

Environment it passes through untouched (set them before calling):
    HF_TOKEN          write token — required unless DEST_PREFIX is set
    HF_UPLOAD_REPO    override the destination repo (default: the source repo)
    DEST_PREFIX       S3 upload root, if you also want a bucket copy
    TOOLS_URL         where build_engines + the .py workers are hosted
    WORKDIR           scratch dir (default: ~/uc-build)

Exit code is the number of FAILED models, so a wrapper can branch on it. A model
that fails does not stop the rest: on a rented box the expensive resource is the
hour, and one bad model should not cost the other four.
"""

import argparse
import json
import os
import platform
import subprocess
import sys
import time
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))


def log(msg):
    print(msg, flush=True)


def detect_sm():
    """`sm_86` from nvidia-smi's compute_cap. The engine name depends on it, so a
    guess is not acceptable - no GPU means no build, not a default."""
    try:
        out = subprocess.check_output(
            ["nvidia-smi", "--query-gpu=compute_cap,name", "--format=csv,noheader"],
            stderr=subprocess.STDOUT, universal_newlines=True)
    except Exception as e:
        raise SystemExit("nvidia-smi failed (%s) - no NVIDIA driver on this VM?" % e)
    first = out.strip().splitlines()[0]
    cap, _, name = first.partition(",")
    return "sm_" + cap.strip().replace(".", ""), name.strip()


def host_os_arch():
    return "win_x64" if platform.system() == "Windows" else "linux_x64"


def load_matrix(path_or_url):
    if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
        with urllib.request.urlopen(path_or_url, timeout=30) as r:
            return json.loads(r.read().decode("utf-8"))
    with open(path_or_url, encoding="utf-8") as f:
        return json.load(f)


def resolve_matrix_path(explicit, tools_url):
    if explicit:
        return explicit
    beside = os.path.join(HERE, "build_matrix.json")
    if os.path.isfile(beside):
        return beside
    if tools_url:
        return tools_url.rstrip("/") + "/build_matrix.json"
    raise SystemExit(
        "build_matrix.json not found beside this script and TOOLS_URL is unset - "
        "pass --matrix <path|url>")


def wrapper_cmd(os_arch):
    """The per-OS wrapper, taken from beside this file (staged together)."""
    if os_arch == "win_x64":
        script = os.path.join(HERE, "build_engines.ps1")
        if not os.path.isfile(script):
            raise SystemExit("build_engines.ps1 not found beside run_matrix.py")
        return ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", script]
    script = os.path.join(HERE, "build_engines.sh")
    if not os.path.isfile(script):
        raise SystemExit("build_engines.sh not found beside run_matrix.py")
    return ["bash", script]


def env_for(entry, arch, base_env):
    """Translate one matrix row into the wrapper's environment contract."""
    cell = entry["trt"][arch]
    env = dict(base_env)
    env["MODEL_ID"] = entry["model_id"]
    env["ROLE"] = entry["role"]
    # "__single__", not "": Windows does not store empty environment variables -
    # SetEnvironmentVariable(name, "") DELETES the variable, so the wrapper sees
    # "unset" and falls back to its "s m l" default. That built three bucketed
    # engines for a single-profile model, under names the daemon never looks up.
    # Both wrappers already treat __single__ as "one engine, no bucket suffix".
    env["BUCKETS"] = " ".join(entry["buckets"]) if entry["buckets"] else "__single__"
    env["PRECISION"] = cell["precision"]
    env["HF_MODEL_REPO"] = entry["hf_repo"]
    # Destination defaults to the source repo - the daemon reads engines from the
    # same place it reads the ONNX. An explicit HF_UPLOAD_REPO still wins.
    env.setdefault("HF_UPLOAD_REPO", entry["hf_repo"])
    if base_env.get("HF_UPLOAD_REPO"):
        env["HF_UPLOAD_REPO"] = base_env["HF_UPLOAD_REPO"]
    return env


def main():
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--matrix", default=None, help="path or URL to build_matrix.json")
    ap.add_argument("--only", action="append", default=[], help="build only this model id (repeatable)")
    ap.add_argument("--skip", action="append", default=[], help="skip this model id (repeatable)")
    ap.add_argument("--arch", default=None, help="override the detected sm_XX (testing only)")
    ap.add_argument("--dry-run", action="store_true", help="print invocations, build nothing")
    args = ap.parse_args()

    tools_url = os.environ.get("TOOLS_URL", "")
    matrix = load_matrix(resolve_matrix_path(args.matrix, tools_url))

    os_arch = host_os_arch()
    if args.arch:
        sm, gpu = args.arch, "(overridden)"
    else:
        sm, gpu = detect_sm()

    log("=" * 70)
    log(" ultracode cloud engine matrix")
    log("   GPU        : %s  ->  %s" % (gpu, sm))
    log("   platform   : %s" % os_arch)
    log("   TRT tag    : %s   (engine names carry it; a mismatch is unloadable)" % matrix["trt_tag"])
    log("=" * 70)

    if sm not in matrix["cuda_archs"]:
        log("")
        log("  !! %s is not in the cloud matrix %s." % (sm, matrix["cuda_archs"]))
        if sm == "sm_120":
            log("     sm_120 is the LOCAL lane - it is built at a desk and already published.")
        else:
            log("     Add it to build_config.cuda_archs and regenerate, or rent a different card.")
        return 1

    base_env = dict(os.environ)
    # The wrapper installs TensorRT before the first build; without the exact
    # version it would either guess or let build_trt_engine.py fail on "cannot
    # determine the TensorRT version".
    if matrix.get("trt_pip_version"):
        base_env["TRT_PIP_VERSION"] = matrix["trt_pip_version"]
    if not base_env.get("HF_TOKEN") and not base_env.get("DEST_PREFIX"):
        log("")
        log("  !! neither HF_TOKEN nor DEST_PREFIX is set - the engines would be built and discarded.")
        return 1

    todo = []
    for entry in matrix["models"]:
        mid = entry["model_id"]
        if args.only and mid not in args.only:
            continue
        if mid in args.skip:
            log("  [skip] %s (--skip)" % mid)
            continue
        # A private HF repo cannot serve the ONNX to a machine holding only a
        # write token for the OTHER repos. Say so once, here, instead of letting
        # the wrapper fail on a 401 five minutes in.
        if entry.get("hf_public") is False and not base_env.get("SRC_ONNX_S3"):
            log("  [skip] %s - HF repo %s is not public and no SRC_ONNX_S3 given"
                % (mid, entry["hf_repo"]))
            continue
        todo.append(entry)

    if not todo:
        log("  nothing to build")
        return 0

    cmd = wrapper_cmd(os_arch)
    results = []
    for entry in todo:
        cell = entry["trt"][sm]
        n = max(1, len(entry["buckets"]))
        log("")
        log("-" * 70)
        log(" %s  [%s]  %d engine(s)  buckets=%s"
            % (entry["model_id"], cell["precision"],
               n, ",".join(entry["buckets"]) or "(single)"))
        log("   source: %s :: %s" % (entry["hf_repo"], cell["source"]["file"]))
        if cell["source"]["ptq_on_vm"]:
            log("   note  : PTQ runs ON THIS VM - the calibration corpus must be staged")
        log("-" * 70)

        env = env_for(entry, sm, base_env)
        if args.dry_run:
            keys = ("MODEL_ID", "ROLE", "BUCKETS", "PRECISION", "HF_MODEL_REPO", "HF_UPLOAD_REPO")
            for k in keys:
                log("   %s=%s" % (k, env.get(k, "")))
            results.append((entry["model_id"], "dry-run", 0.0))
            continue

        t0 = time.time()
        rc = subprocess.call(cmd, env=env)
        dt = time.time() - t0
        results.append((entry["model_id"], "ok" if rc == 0 else "FAILED rc=%d" % rc, dt))
        if rc != 0:
            log("  !! %s failed (rc=%d) - continuing with the rest" % (entry["model_id"], rc))

    log("")
    log("=" * 70)
    log(" SUMMARY  %s / %s" % (sm, os_arch))
    failed = 0
    for mid, status, dt in results:
        mark = "OK  " if status in ("ok", "dry-run") else "FAIL"
        if status.startswith("FAILED"):
            failed += 1
        log("  [%s] %-28s %-14s %6.1fs" % (mark, mid, status, dt))
    log("=" * 70)
    if failed:
        log(" %d model(s) FAILED - engines for those cells are missing." % failed)
    return failed


if __name__ == "__main__":
    raise SystemExit(main())