#!/usr/bin/env python3 """Build TensorRT engine from ONNX model with multi-profile bucketing. Usage: python scripts/build_trt_engine.py [--model-dir PATH] [--cuda-arch sm_120] Two modes: 1. trtexec (if in PATH) — NVIDIA's official CLI tool 2. Python TensorRT API (fallback) — uses tensorrt pip package Profiles (3 buckets matching tensorrt_runtime.zig default_profiles): 0: batch 1-32, seq 1-32 (opt: 16x32) — short entity names 1: batch 1-64, seq 1-64 (opt: 32x64) — entity descriptions 2: batch 1-128, seq 1-128 (opt: 128x128) — docstrings / function bodies Output: {model_id}_{os_arch}_trt{VER}_{cuda_arch}.engine in the model directory (os_arch = win_x64 / linux_x64 / linux_arm64 / mac_arm64; VER = TensorRT major.minor from vendor/tensorrt/VERSION, e.g. trt11.0). A serialized plan is keyed on {GPU arch x OS x TRT version} and TensorRT offers no compat fallback on deserialize, so the version belongs in the filename next to the arch: the daemon then resolves an engine it can actually load instead of opening a same-named plan from another TRT and failing inside the runtime. """ import argparse import os import shutil import subprocess import sys import warnings from pathlib import Path # Force UTF-8 output on Windows (default console encoding is CP1251/CP1252) if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8", errors="replace") if hasattr(sys.stderr, "reconfigure"): sys.stderr.reconfigure(encoding="utf-8", errors="replace") # Auto-install pip deps on first run when this script is invoked directly # (i.e. without going through models/code-daemon-embed/run.ps1 / run.sh which # already install everything). Idempotent — skips packages that are already importable. if os.environ.get("ULTRACODE_NO_BOOTSTRAP") != "1": # Repo root by walking up to build.zig, not by counting `..` — this file has # moved once already (scripts/ -> models/_compile/backends/workers/) and a # hardcoded depth breaks silently, since a missing directory just means "no # bootstrap available". _root = Path(__file__).resolve() for _ in range(6): _root = _root.parent if (_root / "build.zig").is_file(): break _bootstrap_dir = _root / "models" / "code-daemon-embed" if (_bootstrap_dir / "_bootstrap.py").exists(): sys.path.insert(0, str(_bootstrap_dir)) try: from _bootstrap import ensure_deps, ensure_nvidia_deps # onnx is needed for ensure_single_file_onnx() and proto inspection; # torch is not required by this script — skip the torch install. ensure_deps(extra=["onnx"], torch_required=False) # tensorrt + cuda-python (NVIDIA-only). No-op on macOS / CPU hosts. ensure_nvidia_deps() except Exception as _bs_err: print(f"[bootstrap] WARNING: {_bs_err}", flush=True) finally: sys.path.remove(str(_bootstrap_dir)) # Suppress TRT deprecation warnings: implicit INT8 calibration (superseded by QAT/explicit # quantization in TRT 10.1+) and per-layer precision (superseded by strong typing in TRT 10.12+). # Replacements require ONNX QAT nodes / STRONGLY_TYPED mode — not applicable here. warnings.filterwarnings("ignore", message=".*Deprecated.*", category=DeprecationWarning) warnings.filterwarnings("ignore", message=".*deprecated.*", category=DeprecationWarning) def ensure_single_file_onnx(onnx_path): """Convert split ONNX (model.onnx + model.onnx_data) to single-file if needed.""" data_file = onnx_path + "_data" if not os.path.exists(data_file): return onnx_path merged_path = onnx_path.replace(".onnx", "_merged.onnx") if os.path.exists(merged_path): merged_size = os.path.getsize(merged_path) if merged_size < 1024 or merged_size > 2_000_000_000: # Empty/corrupted or >2GB (protobuf can't parse) — delete stale cache os.remove(merged_path) print(f" Removed stale merged ONNX ({merged_size / 1e6:.0f} MB)") else: merged_mtime = os.path.getmtime(merged_path) if (merged_mtime > os.path.getmtime(onnx_path) and merged_mtime > os.path.getmtime(data_file)): print(f" Split ONNX: using cached merged file") return merged_path try: import onnx print(f" Split ONNX detected: merging into single file...") model = onnx.load(onnx_path, load_external_data=True) onnx.save(model, merged_path) merged_size = os.path.getsize(merged_path) if merged_size > 2_000_000_000: # >2GB: protobuf can't parse it back, trtexec may fail. Use original + external data. os.remove(merged_path) print(f" Merged {merged_size / 1e6:.0f} MB > 2GB limit, using split ONNX (parse_from_file handles external data)") return onnx_path print(f" Merged: {merged_size / (1024*1024):.1f} MB") return merged_path except Exception as e: print(f" Split ONNX merge failed: {e}") return onnx_path # Profile specs: (max_batch, max_seq, opt_batch, opt_seq) # # Single rectangular profile per model type — no multi-profile bucketing. # The daemon accumulates up to batch=128 before dispatch; buildEmbedInput # caps at 420 chars ≈ 64 tokens so seq=64 covers all real traffic. # Fewer profiles = less VRAM, faster compile, trivial selectProfile (always 0). # Multi-profile builds were tried and produce slower kernels for the seq=64 path # on Gemma sm_120 (autotuner picks kernels optimized for the broader shape range). # # Profile specs: (max_batch, max_seq, opt_batch, opt_seq). # # 2026-05-05 — half-batch-zero saga resolved. The "TRT sm_120 regression" # we chased turned out to be a Zig-side dtype mismatch: engine input # binding is INT64 (verified via getTensorDtype), but our worker copied # host buffers as INT32 stride. Engine read 8 bytes per token from a # buffer where we'd written 4 → got 2× rows of garbage from uninit # memory, manifesting as "second half of every batch is zero". Fixed # in tensorrt_runtime.zig + embed_worker.zig (i32 → i64 throughout the # TRT input chain). Engine itself is fine; restore full 128 batch. PROFILES = [ (128, 64, 128, 64), # 0: rectangular fast path — batch 128, seq 64 ] RECTANGULAR_PROFILE = True # Dynamic-seq probe (2026-07-28). Set `DYNSEQ_OVERRIDE="batch,min_seq,opt_seq,max_seq"` # to build ONE engine whose SEQUENCE dimension is a range while the batch stays # rectangular. Purpose: a chunk could then be dispatched at its own longest text # instead of the bucket's ceiling — live telemetry says 15.3% of GPU token-work # is padding (avg 42/48, 56/64, 70/80, 95/128). The question this answers is # whether TRT's broader-range kernel choice costs more than the padding it saves; # the file header already records that multi-PROFILE builds regressed seq=64 on # sm_120. Applied AFTER --variant in main() so the engine still gets a suffixed # filename. Production builds leave it unset. DYNSEQ_MIN_SEQ = None # ─── Experimental switches (hardcoded) ────────────────────────────────── # Project preference: in-source const flip is cleaner than env-var dance # for one-off experiments. Each switch documents its evidence + revert. # Flip the value, save, run models/compile or build_trt_engine directly. # Builder workspace pool. Default 4 GB; bump to 8-12 GB to widen TRT # tactic search on autotuning. Build-time only — does NOT affect engine # size or runtime VRAM. Risk: build OOM if value > free VRAM at build time. # REVERT: change back to 4. TRT_WORKSPACE_GB = 8 # TRT max_aux_streams config. # 0 — workaround for the 2026-04-28 "5 valid + N zeros" sm_120 codegen # bug (TRT 10.13.3). Forces single-stream kernel generation. # 4 — TRT default. Researcher confirmed 2026-05-19 that post-fusion # embedding graphs (BGE/Qwen3/Gemma) are too linear for aux streams # to help, and TRT 10.16 gracefully self-disables ("Multi stream is # disabled because the stream assignment failed" warning). Either # value produces functionally identical engines on this family. # Kept as probe knob — REVERT: 0. TRT_MAX_AUX_STREAMS = 0 # Env override for sweep experiments. Format: RECT_PROFILE_OVERRIDE="batch,seq" # — collapses PROFILES to a single (batch, seq, batch, seq) entry. Production # builds leave this unset. Example: RECT_PROFILE_OVERRIDE="32,32". _rect_override = os.environ.get("RECT_PROFILE_OVERRIDE", "").strip() if _rect_override: try: _b, _s = (int(x) for x in _rect_override.split(",")) PROFILES = [(_b, _s, _b, _s)] print(f"[build_trt] PROFILES overridden via env: rectangular ({_b}, {_s})", flush=True) except (ValueError, AttributeError) as _e: print(f"[build_trt] WARN: bad RECT_PROFILE_OVERRIDE='{_rect_override}': {_e}", flush=True) # Length-aware bucket variants — when `--variant {s,m,l}` is passed, PROFILES # collapses to that variant's rectangular (max_batch, max_seq). Single source # of truth lives in `models/code-daemon-embed/embed_bucket_config.py`. We # duplicate the tuples here to avoid forcing the build to import from a model # subdirectory at parse-time (the bootstrap path may not yet be on sys.path). # `models/_compile/preflight_bucket_sync.py` runs as a pre-build gate to catch drift. BUCKET_VARIANT_PROFILES = { # 2026-07-27: grid 40/64/128 → 32/48/80/128, measured on real engines # (see the header comment in src/inference/embed_bucket_config.zig). "s": (96, 48), "m": (128, 64), "l": (128, 80), "xl": (256, 128), } # (min_seq, opt_seq) per embed bucket — the SEQUENCE dimension is a range while # the batch stays rectangular, so a chunk is dispatched at its own longest text # instead of the bucket ceiling. Live telemetry put 15.3% of GPU token-work in # sequence padding; recovering it measured +10.2% end to end (9,738 -> 10,731 # emb/s on mysql-server) and is quality-neutral by construction — padding is # masked in attention, verified at cosine 1.000000 on 128 rows of real ids. # # Two rules produced these numbers and neither is guessable: # - `opt` sits at the bucket's histogram-weighted MEAN, not its ceiling. The # tax is paid for being far from the opt point, not for being dynamic: one # wide 16→64→128 engine costs +0.8% at seq 64 and +15.4% at seq 80. # - `min` is the bucket FLOOR, not the observed corpus minimum — `s` gets 8 # rather than the measured 24, because the router must accept a 3-token text. # # This lived in a DYNSEQ_OVERRIDE env var and was deployed by copying files over # the rectangular ones. A later `models/compile` run rebuilt them rectangular and # the +10.2% was silently gone; the engines still loaded, still returned 768-dim # vectors, and reported 54 layers instead of 69 where nobody was looking. Hence # the table: a plain build now produces the shape that is meant to ship. BUCKET_VARIANT_DYNSEQ = { "s": (8, 48), "m": (56, 56), "l": (72, 72), "xl": (88, 96), } # NER single profile: batch=32 (NER worker --batch-size 32), seq=128. # Bumped from seq=64 (2026-05-05) — ~20% of incoming chunks exceed 64 # tokens (long entity descriptions, function bodies, docstrings), causing # `SetShapeFailed` at runtime and DocGraph falling back to rules-only for # those entities. The OV NER variants ship as `_b32_s128`, so the ONNX # already supports seq=128. Opt point sits at 96 — middle of the 1..128 # range, biasing the autotuner toward the typical mass of inputs without # regressing the long tail. NER_PROFILES = [ # 0: NER main path — RECTANGULAR (max == opt) to avoid an invalid # min<=opt<=max profile. The previous value (32, 128, 32, 96) set # opt_seq=96 < max_seq=128, which under RECTANGULAR_PROFILE (min==max==128) # makes opt_seq < min_seq → TRT 11 rejects with "Invalid optimization # profiles ... ensure MIN<=OPT<=MAX". The NER worker always pads to # batch=32 seq=128, so a static 32x128 profile is correct. (32, 128, 32, 128), ] # Reranker single profile: batch=16 (reranker worker max_batch=16, matches the OV engine # `_b16_s256` and models/compile.cmd's $ovChkBatch=16), seq=256. The cross-encoder scores # (query, document) PAIRS, which are long, so seq must be 256 — NOT the embedding default # (128, 64). Without a reranker-specific profile, `--role reranker` fell through to the # embedding PROFILES (128, 64); the engine bound seq=64 while handleRerank pads to # max_seq_len=256 → SetShapeFailed at runtime → search degraded to no-rerank. RECTANGULAR # (min==opt==max) per handleRerank's rectangular-engine contract (embed_worker.zig:4867). RERANKER_PROFILES = [ (16, 256, 16, 256), ] def _installed_trt_tag(): """`trt{major}.{minor}` of the TensorRT in THIS interpreter, or None.""" try: import tensorrt as trt parts = str(trt.__version__).split(".") except Exception: return None return f"trt{parts[0]}.{parts[1]}" if len(parts) >= 2 and parts[0] else None def _pinned_trt_tag(): """`trt{major}.{minor}` from `vendor/tensorrt/VERSION`, or None if no repo. This is the pin build.zig turns into the daemon's `runtime_versions.trt_tag`, so naming from it guarantees the engine we write is the engine the worker looks for. Absent when the script runs standalone — the cloud sm_86/sm_89 builder (models/_cloud_shared/build-tools/build_engines.sh) copies only this .py onto a bare VM, with no vendor/ tree to read. """ d = os.path.dirname(os.path.abspath(__file__)) for _ in range(6): # walk up to repo root pin = os.path.join(d, "vendor", "tensorrt", "VERSION") if os.path.isfile(pin): try: parts = open(pin).read().strip().split(".") except OSError: return None return f"trt{parts[0]}.{parts[1]}" if len(parts) >= 2 and parts[0] else None parent = os.path.dirname(d) if parent == d: return None d = parent return None def _trt_runtime_tag(): """Runtime-version tag embedded in every engine filename, e.g. `trt11.0`. Prefers the repo pin (what the DAEMON will deserialize with); falls back to the TensorRT actually doing the build, so a standalone/cloud run still produces a correctly-labelled engine instead of failing. Only when neither is knowable do we stop: an untagged engine is invisible to the daemon, and TRT builds are far too slow to discover that afterwards. """ tag = _pinned_trt_tag() or _installed_trt_tag() if tag: return tag print("ERROR: cannot determine the TensorRT version — neither vendor/tensorrt/VERSION") print(" nor an importable `tensorrt` package. Engine filenames carry the") print(" TensorRT major.minor and the daemon only loads its own version.") sys.exit(1) def _refuse_trt_version_skew(rt_tag): """Stop when the TensorRT building this engine is not the pinned one. Matters more here than anywhere else: the plan is produced by the `tensorrt` package in THIS interpreter but deserialized by the vendored nvinfer the daemon links. If they differ, the file gets a name promising a runtime that cannot read it — a load failure the version tag exists to prevent. This was a WARNING until 2026-09-12. It printed on every Linux build for weeks while every Linux engine was built by 11.2 and named trt11.0 (`models/_compile/deps/wsl.py:_trt_pin`), because nothing consumed it. There is no legitimate skew: a patch-level difference yields the same tag and never reaches this branch, and a standalone cloud run has no pin, so `rt_tag` IS the installed tag there. A missing package is handled elsewhere. """ installed_tag = _installed_trt_tag() if installed_tag and installed_tag != rt_tag: print(f"ERROR: TensorRT version skew — this interpreter has {installed_tag} " f"but the engine would be named {rt_tag} (from vendor/tensorrt/VERSION).") print(f" The daemon deserializes with the pinned runtime and would " f"REJECT the plan. Install tensorrt-cu13 at the pin, or bump the pin.") sys.exit(1) def get_cuda_arch(): """Detect CUDA compute capability via nvidia-smi.""" try: out = subprocess.check_output( ["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"], text=True, ).strip().split("\n")[0] major, minor = out.split(".") return f"sm_{major}{minor}" except Exception: return None def _onnx_is_valid(path): """Quick-check that an ONNX file is parseable (not truncated). Files >1 MB that fail onnx.load are assumed valid: Python protobuf has a 2 GB parse limit but trtexec uses the C++ ONNX parser which has no such limit.""" try: import onnx onnx.load(path, load_external_data=False) return True except Exception as e: try: size = os.path.getsize(path) if size > 1024 * 1024: print(f" NOTE: onnx.load failed ({e}), file is {size / (1024*1024):.0f} MB " f"— assuming valid (trtexec uses C++ parser, no 2 GB limit)") return True except Exception: pass return False def _has_all_external_data(onnx_path): """Return True if every external-data file referenced by the ONNX header exists in the same directory. Returns True (optimistic) when the header itself cannot be loaded (e.g. 2 GB+ merged file — no external refs to check).""" try: import onnx as _onnx header = _onnx.load(onnx_path, load_external_data=False) except Exception: return True # can't check — assume ok (2 GB+ merged, no external refs) onnx_dir = os.path.dirname(os.path.abspath(onnx_path)) for t in header.graph.initializer: if t.data_location == _onnx.TensorProto.EXTERNAL: for entry in t.external_data: if entry.key == "location": if not os.path.exists(os.path.join(onnx_dir, entry.value)): return False return True def find_onnx(model_dir, mode="int8"): """Find ONNX model in model directory, with validation and cache fallback. `mode` controls preference order: - "fp8": prefer `model_fp8qdt.onnx` (ModelOpt FP8 PTQ output) over INT8. - "int8" (default): prefer `model_int8qdt.onnx` (ModelOpt INT8 PTQ output). - "bf16": prefer plain `model.onnx`. Falls back through the alternate-precision PTQ ONNX before raw `model.onnx` so that a misconfigured catalog still produces a build (TRT will pick up whatever Q/DQ nodes are in the file). """ # Last-resort fallback: the compiler's download cache for third-party models # (`models/_compile/.cache/models//`). The compiler itself always passes # `--onnx` explicitly and the cloud builder stages the ONNX into --model-dir, # so this is only reached by a hand-run invocation. # # It used to be derived as `/.cache/`, which pointed # at `scripts/.cache` while the script lived there and became a path that can # never exist once it moved. model_name = os.path.basename(os.path.normpath(model_dir)) _pkg_dir = os.path.dirname(os.path.abspath(__file__)) # …/backends/workers _compile_root = os.path.dirname(os.path.dirname(_pkg_dir)) # …/models/_compile cache_dir = os.environ.get("ULTRACODE_COMPILE_CACHE") or \ os.path.join(_compile_root, ".cache") cache_path = os.path.join(cache_dir, "models", model_name, "model.onnx") int8_candidates = [ os.path.join(model_dir, "onnx", "model_int8qdt.onnx"), os.path.join(model_dir, "model_int8qdt.onnx"), os.path.join(cache_dir, "models", model_name, "model_int8qdt.onnx"), ] fp8_candidates = [ os.path.join(model_dir, "onnx", "model_fp8qdt.onnx"), os.path.join(model_dir, "model_fp8qdt.onnx"), os.path.join(cache_dir, "models", model_name, "model_fp8qdt.onnx"), ] plain_candidates = [ os.path.join(model_dir, "model.onnx"), os.path.join(model_dir, "onnx", "model.onnx"), cache_path, ] if mode == "fp8": candidates = fp8_candidates + int8_candidates + plain_candidates elif mode == "bf16": candidates = plain_candidates + int8_candidates + fp8_candidates else: candidates = int8_candidates + plain_candidates + fp8_candidates for p in candidates: if os.path.exists(p): if not _onnx_is_valid(p): print(f" WARNING: {p} is corrupted (truncated?), trying next candidate") continue if not _has_all_external_data(p): print(f" NOTE: {p} has missing external data files — trying next candidate") continue return p return None def onnx_has_qdq(onnx_path): """True if the ONNX model contains explicit Q/DQ nodes (PTQ quantized). When Q/DQ nodes are present, TRT must have a matching precision builder flag enabled (INT8 for int8qdt, FP8 for fp8qdt). Without it, TRT silently ignores DequantizeLinear/QuantizeLinear and builds a plain BF16/FP16 engine, wasting the calibration work. Detection by filename is sufficient — quantize_onnx.py always writes 'model_int8qdt.onnx' (INT8 mode) or 'model_fp8qdt.onnx' (FP8 mode). """ name = os.path.basename(onnx_path) return any(tag in name for tag in ("int8qdt", "fp8qdt", "nvfp4qdt", "int4qdt")) def onnx_is_fp8(onnx_path): """True if the ONNX model is a ModelOpt FP8 PTQ output (fp8qdt naming). Used to distinguish FP8-quantized ONNX (needs --fp8 builder flag) from INT8-quantized ONNX (needs --int8). When neither, fallback paths apply. """ return "fp8qdt" in os.path.basename(onnx_path) def build_with_trtexec(onnx_path, engine_path, trtexec_bin="trtexec"): """Build engine using trtexec CLI (TRT 11: strongly-typed only).""" onnx_path = ensure_single_file_onnx(onnx_path) cmd = [trtexec_bin, f"--onnx={onnx_path}", f"--saveEngine={engine_path}"] # TRT 11: precision is read entirely from the ONNX Q/DQ nodes. Strongly typed # is the only typing mode — the per-precision flags (--int8/--fp16/--bf16/--fp8) # and the INT8 calibrator (--calib) were removed. `--stronglyTyped` is not # passed: TRT 11.3 release notes list it as a flag with no effect. # Timing cache: speeds up rebuilds dramatically (minutes → seconds) timing_cache = os.path.join(os.path.dirname(engine_path), "timing.cache") cmd.append(f"--timingCacheFile={timing_cache}") # Max builder optimization level for best engine quality cmd.append("--builderOptimizationLevel=5") # 2026-04-28 — sm_120 aux-stream regression fix (researcher rec #4 in # .autodoc/archive/inference-backends.md). TRT 10.13.3 / 10.16.1 release notes # document a multi-stream codegen bug on consumer Blackwell (sm_120) # that affects pooling/attention subgraphs and produces partial # outputs (the embeddinggemma "5 valid + N-5 zeros" symptom matches # this signature: kernel runs but writes zeros to trailing rows). # Disabling aux streams bypasses the buggy codegen path; cost is # one stream of dispatch concurrency we don't currently exploit. # # 2026-05-19 — value lives at module top as TRT_MAX_AUX_STREAMS const. # Flip there + rebuild to probe (researcher confirmed it's effectively # a no-op on embedding models; TRT 10.16 self-disables anyway). cmd.append(f"--maxAuxStreams={TRT_MAX_AUX_STREAMS}") if TRT_MAX_AUX_STREAMS != 0: print(f" [build_trt] TRT_MAX_AUX_STREAMS={TRT_MAX_AUX_STREAMS} — sm_120 workaround DISABLED") # `--useCudaGraph` / `--useSpinWait` were passed here until 2026-09-22. They # only shaped trtexec's post-build benchmark, never the saved engine, and # TRT 11.3 release notes list both as flags with no effect — so a benchmark # line printed by an 11.3 trtexec is measured WITHOUT graphs and does not # compare with 10.16/11.0 logs. The runtime counterpart was measured and # left off: `cuda_graphs_enabled` in src/inference/tensorrt_runtime.zig. for max_b, max_s, opt_b, opt_s in PROFILES: cmd.extend([ f"--minShapes=input_ids:1x1,attention_mask:1x1", f"--optShapes=input_ids:{opt_b}x{opt_s},attention_mask:{opt_b}x{opt_s}", f"--maxShapes=input_ids:{max_b}x{max_s},attention_mask:{max_b}x{max_s}", ]) print(f" trtexec command: {' '.join(cmd[:5])} ...") result = subprocess.run(cmd) return result.returncode == 0 def _add_nvinfer_to_path(): """Add vendor/tensorrt/lib AND pip tensorrt_libs to DLL search path.""" added = [] # 1. vendor/tensorrt/lib (our bundled runtime DLLs). # This file lives at models/_compile/backends/workers/, so the repo root is # four levels up. It used to be one (`scripts/../vendor`), which stopped # resolving the moment the script moved — silently, because the directory # simply "was not there" and the pip tensorrt_libs below usually covered it. _here = os.path.dirname(os.path.abspath(__file__)) _repo_root = os.path.abspath(os.path.join(_here, "..", "..", "..", "..")) vendor_lib = os.path.join(_repo_root, "vendor", "tensorrt", "lib") if os.path.isdir(vendor_lib): added.append(vendor_lib) # 2. pip tensorrt_libs (contains nvinfer_plugin_10.dll needed for builder) try: import site for sp in site.getsitepackages() + [site.getusersitepackages()]: trt_libs = os.path.join(sp, "tensorrt_libs") if os.path.isdir(trt_libs): added.append(trt_libs) break except Exception: pass # Register all paths for lib_dir in added: if sys.platform == "win32" and hasattr(os, "add_dll_directory"): try: os.add_dll_directory(lib_dir) except OSError: pass os.environ["PATH"] = lib_dir + os.pathsep + os.environ.get("PATH", "") return added[0] if added else None def _import_tensorrt(): """Import tensorrt, handling the cu12_bindings layout.""" _add_nvinfer_to_path() # Try direct import first (tensorrt meta-package installed) try: import tensorrt return tensorrt except (ImportError, ModuleNotFoundError, FileNotFoundError, OSError): pass # Try bindings-only import (tensorrt_cu12_bindings installed) try: from tensorrt_bindings import tensorrt return tensorrt except (ImportError, ModuleNotFoundError, FileNotFoundError, OSError): pass return None def _has_cuda_runtime(): """Check if CUDA runtime is available (libcudart loadable).""" import ctypes for lib in ["libcudart.so", "libcudart.so.13", "libcudart.so.12", "libcudart.so.11", "cudart64_13.dll", "cudart64_12.dll", "cudart64_110.dll"]: try: ctypes.cdll.LoadLibrary(lib) return True except OSError: pass # Also check if nvcc is in PATH (CUDA toolkit installed) if shutil.which("nvcc"): return True return False def ensure_tensorrt_package(): """Auto-install tensorrt Python bindings + libs if missing.""" trt = _import_tensorrt() if trt is not None: return True if not _has_cuda_runtime(): print(" tensorrt not available: CUDA runtime not found (no libcudart/nvcc)", file=sys.stderr) print(" Install CUDA toolkit or add trtexec to PATH", file=sys.stderr) return False print(" tensorrt not found — installing via pip...") nvidia_idx = "https://pypi.nvidia.com" # Detect CUDA major version to pick matching tensorrt build. Mixing cu12 # and cu13 packages produces version-mismatch errors at engine deserialize # because the engine plan encodes the build-time native DLL version, and # `import tensorrt` then resolves to a different package's DLL at runtime. cuda_major = None try: nvsmi = subprocess.check_output( ["nvidia-smi"], stderr=subprocess.DEVNULL, text=True, timeout=5, ) import re m = re.search(r"CUDA Version:\s*(\d+)\.(\d+)", nvsmi) if m: cuda_major = int(m.group(1)) except Exception: pass if cuda_major and cuda_major >= 13: candidates = [ ["tensorrt-cu13-bindings==11.0.0.114", "tensorrt-cu13-libs==11.0.0.114"], ["tensorrt-cu13-bindings", "tensorrt-cu13-libs"], ] else: candidates = [ ["tensorrt-cu12-bindings==11.0.0.114", "tensorrt-cu12-libs==11.0.0.114"], ["tensorrt-cu12-bindings", "tensorrt-cu12-libs"], ] for pkgs in candidates: result = subprocess.run( [sys.executable, "-m", "pip", "install", "--quiet", "--only-binary", ":all:", "--extra-index-url", nvidia_idx] + pkgs, capture_output=True, text=True, ) if result.returncode == 0: break else: # Last resort: try the meta-package subprocess.run( [sys.executable, "-m", "pip", "install", "--quiet", "--only-binary", ":all:", "--extra-index-url", nvidia_idx, "tensorrt"], capture_output=True, text=True, ) trt = _import_tensorrt() if trt is not None: print(f" tensorrt {trt.__version__} ready") return True print(" tensorrt still not importable after install", file=sys.stderr) return False def _upgrade_onnx_opset(onnx_path, target_opset=17): """Upgrade ONNX model to target opset if needed (fixes LayerNorm FP16 overflow).""" # Handle split ONNX first — skip upgrade if merge fails (>2GB models) data_file = onnx_path + "_data" if os.path.exists(data_file): merged = ensure_single_file_onnx(onnx_path) if merged == onnx_path: # Merge failed (protobuf 2GB limit) — skip opset upgrade, TRT will parse directly print(f" Opset upgrade skipped: split ONNX too large to merge") return onnx_path onnx_path = merged else: onnx_path = ensure_single_file_onnx(onnx_path) try: import onnx from onnx import version_converter model = onnx.load(onnx_path) current_opset = model.opset_import[0].version if model.opset_import else 0 if current_opset >= target_opset: return onnx_path # already sufficient print(f" Upgrading ONNX opset {current_opset} → {target_opset} (fixes LayerNorm FP16 overflow)") model = version_converter.convert_version(model, target_opset) upgraded_path = onnx_path.replace(".onnx", f"_opset{target_opset}.onnx") onnx.save(model, upgraded_path) return upgraded_path except Exception as e: print(f" ONNX opset upgrade skipped: {e}") return onnx_path def build_with_python_api(onnx_path, engine_path, fp16=True, int8=False, fp8=False, bf16=False, per_layer_norm=False, strongly_typed=False): """Build engine using TensorRT Python API (no trtexec needed).""" if not ensure_tensorrt_package(): return False trt = _import_tensorrt() # Upgrade ONNX opset to 17+ for native INormalizationLayer (fixes FP16 LayerNorm overflow) onnx_path = _upgrade_onnx_opset(onnx_path, target_opset=17) logger = trt.Logger(trt.Logger.WARNING) builder = trt.Builder(logger) # TRT 11: EXPLICIT_BATCH was removed (explicit batch is the only mode) and # STRONGLY_TYPED is the default + only typing mode. Precision comes entirely # from the ONNX Q/DQ nodes — TRT reads each tensor's data_type as-is, with no # backward type-inference that would propagate INT8 onto a DequantizeLinear's # shared scale tensor. Per-precision BuilderFlags, the INT8 calibrator, and # per-layer precision were ALL removed in TRT 11; this is now the only path. network_flags = 1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED) network = builder.create_network(network_flags) print(" STRONGLY_TYPED: enabled (TRT 11 default; precision from Q/DQ ONNX nodes)") parser = trt.OnnxParser(network, logger) print(f" Parsing ONNX: {onnx_path}") # Use parse_from_file if available (TRT 8.6+) — resolves external data (model.onnx_data) # relative to the ONNX file directory. Falls back to parse(bytes) + chdir for older TRT. parsed_ok = False if hasattr(parser, "parse_from_file"): parsed_ok = parser.parse_from_file(onnx_path) else: # Older TRT: chdir to model directory so external data is found prev_cwd = os.getcwd() os.chdir(os.path.dirname(os.path.abspath(onnx_path))) try: with open(onnx_path, "rb") as f: parsed_ok = parser.parse(f.read()) finally: os.chdir(prev_cwd) if not parsed_ok: for i in range(parser.num_errors): print(f" ONNX parse error: {parser.get_error(i)}", file=sys.stderr) return False config = builder.create_builder_config() # Workspace pool — bigger = wider tactic search. Const TRT_WORKSPACE_GB # at module top. Default 4 GB; bumping to 8-12 GB widens tactic search. # Build-time only — freed when builder exits. config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, TRT_WORKSPACE_GB << 30) print(f" Workspace pool: {TRT_WORKSPACE_GB} GB" + (" (default)" if TRT_WORKSPACE_GB == 4 else " (probe value)")) # Max builder optimization level for best engine quality config.builder_optimization_level = 5 # Detailed profiling info — needed for the engine_inspector dump below # to enumerate fused layers (MHA, GEMM, SkipLayerNormalization, etc). # Negligible build-time overhead; engine itself is unchanged. try: config.profiling_verbosity = trt.ProfilingVerbosity.DETAILED except AttributeError: pass # 2026-04-28 — sm_120 aux-stream regression fix (researcher rec #4). # TRT 10.13.3 / 10.16.1 release notes document a multi-stream # codegen bug on consumer Blackwell (sm_120). Setting max_aux_streams # to 0 bypasses the buggy path. Property added in TRT 8.6+; older # versions silently ignore (try/except for safety). # # 2026-05-19 — value from module-top const TRT_MAX_AUX_STREAMS. try: config.max_aux_streams = TRT_MAX_AUX_STREAMS if TRT_MAX_AUX_STREAMS == 0: print(" max_aux_streams: 0 (sm_120 regression workaround)") else: print(f" max_aux_streams: {TRT_MAX_AUX_STREAMS} (probe value — TRT may auto-disable on linear graphs)") except AttributeError: pass # Precision: TRT 11 reads every tensor type from the strongly-typed network # (the ONNX Q/DQ nodes). The per-precision BuilderFlags (INT8/FP16/BF16/FP8), # the INT8 calibrator (IInt8EntropyCalibrator2 + int8_calibrator + # set_calibration_profile), and per-layer precision (layer.precision / # set_output_type) were ALL removed in TRT 11 — there are no builder flags to # set here. The INT8 W8A16 / FP8 E4M3 / NVFP4 path is selected purely by which # *_qdt.onnx was produced by quantize_onnx.py. print(" Precision: from ONNX Q/DQ nodes (strongly-typed; no builder flags)") # Timing cache: speeds up rebuilds dramatically timing_cache_path = os.path.join(os.path.dirname(engine_path), "timing.cache") if os.path.exists(timing_cache_path): with open(timing_cache_path, "rb") as f: cache = config.create_timing_cache(f.read()) print(f" Timing cache: loaded ({os.path.getsize(timing_cache_path)} bytes)") else: cache = config.create_timing_cache(b"") config.set_timing_cache(cache, ignore_mismatch=False) # Create optimization profiles for i, (max_b, max_s, opt_b, opt_s) in enumerate(PROFILES): profile = builder.create_optimization_profile() # RECTANGULAR_PROFILE: min == opt == max. TRT autotuner picks a # single set of tactics for the exact (batch, seq) shape — no # variable-batch search, no autotuner regression on sm_120. # Trapezoid mode: min=(1,1) → opt → max. Lets TRT accept smaller # real batches at runtime, but autotuner must cover the whole # range and on consumer Blackwell that range produces broken # kernels for the upper half of the batch dimension. if DYNSEQ_MIN_SEQ is not None: # Batch stays rectangular (the sm_120 batch-range codegen problem # above is untouched); only seq becomes a range. min_b, min_s = max_b, DYNSEQ_MIN_SEQ elif RECTANGULAR_PROFILE: min_b, min_s = max_b, max_s else: min_b, min_s = 1, 1 profile.set_shape("input_ids", (min_b, min_s), (opt_b, opt_s), (max_b, max_s)) profile.set_shape("attention_mask", (min_b, min_s), (opt_b, opt_s), (max_b, max_s)) # token_type_ids if present in the model for j in range(network.num_inputs): inp = network.get_input(j) if inp.name == "token_type_ids": profile.set_shape("token_type_ids", (min_b, min_s), (opt_b, opt_s), (max_b, max_s)) break config.add_optimization_profile(profile) shape_kind = "RECTANGULAR" if RECTANGULAR_PROFILE else "trapezoid" print(f" Profile {i}: {shape_kind} {min_b}x{min_s} → {opt_b}x{opt_s} → {max_b}x{max_s}") print(f" Building engine (this may take several minutes)...") serialized = builder.build_serialized_network(network, config) if serialized is None: print("ERROR: Engine build failed", file=sys.stderr) return False os.makedirs(os.path.dirname(engine_path), exist_ok=True) with open(engine_path, "wb") as f: f.write(serialized) # Save timing cache for future rebuilds with open(timing_cache_path, "wb") as f: f.write(cache.serialize()) print(f" Timing cache saved: {timing_cache_path}") # Layer-info sidecar (for MHA fusion verification, autotuner tactic # inspection). Deserialize the just-saved engine, dump full JSON of # all layers via engine_inspector. Needs profiling_verbosity=DETAILED # at build time (set above) for tactic names + dtypes to be present. layers_json_path = engine_path.replace(".engine", ".layers.json") try: runtime = trt.Runtime(trt.Logger(trt.Logger.WARNING)) eng = runtime.deserialize_cuda_engine(serialized) if eng is not None: inspector = eng.create_engine_inspector() layer_json = inspector.get_engine_information(trt.LayerInformationFormat.JSON) with open(layers_json_path, "w", encoding="utf-8") as fj: fj.write(layer_json) n_layers = layer_json.count('"LayerType"') n_int8 = layer_json.count('"Int8"') n_fp8 = layer_json.count('"Fp8"') + layer_json.count('"FP8"') n_bf16 = layer_json.count('"BFloat16"') + layer_json.count('"BF16"') # Half and Float were missing from this verdict, so an all-FP32 engine and # a fully-FP16 one printed the identical "Int8=0 BF16=0 FP8=0" line. That # blind spot is how the FP16 lane shipped FP32 engines unnoticed — the one # report that could have caught it did not look at the two types involved. n_half = layer_json.count('"Half"') n_float = layer_json.count('"Float"') # TRT 11's Myelin names the fused kernel `_gemm_mha_v2_*`; the two older # spellings never appear in a TRT 11 dump, so counting only them printed # NOT DETECTED for every engine on 2026-09-11 - including all eleven # XLM-R engines that carry 24 of these kernels each. lj = layer_json.lower() n_fmha = lj.count('fmha') + lj.count('fused_multi_head_attention') + lj.count('_gemm_mha') print(f" Layer info: {layers_json_path}") print(f" {n_layers} layers — Half={n_half} Float={n_float} " f"Int8={n_int8} BF16={n_bf16} FP8={n_fp8}") if n_half == 0 and n_int8 == 0 and n_bf16 == 0 and n_fp8 == 0 and n_float > 0: print(" WARNING: engine is ENTIRELY FP32 — no reduced precision took " "effect. TRT 11 reads precision from the ONNX types only.") if n_fmha > 0: print(f" MHA fusion: DETECTED ({n_fmha} fmha tactic refs)") else: print(f" MHA fusion: NOT DETECTED — consider ONNX-level attention export") except Exception as e: print(f" Layer-info dump SKIPPED: {e}") return True def main(): parser = argparse.ArgumentParser(description="Build TRT engine from ONNX") parser.add_argument( "--model-dir", default="dist/models/multilingual-e5-small", help="Path to model directory containing model.onnx", ) parser.add_argument("--cuda-arch", default=None, help="CUDA arch (e.g. sm_120)") parser.add_argument("--fp16", action="store_true", default=True, help="Enable FP16") parser.add_argument("--int8", action="store_true", default=True, help="Enable INT8 mixed precision") parser.add_argument("--no-int8", action="store_true", help="Disable INT8 (FP16 only)") parser.add_argument("--fp8", action="store_true", help="Enable FP8 (E4M3) — Hopper/Blackwell only. Implies --no-int8.") parser.add_argument("--bf16", action="store_true", help="Enable BF16 (FP32-range half-storage) — required for Gemma RMSNorm. Disables FP16/INT8/FP8.") parser.add_argument("--per-layer-norm", action="store_true", help="Per-layer precision: norm layers → BF16, rest → INT8. " "Requires --int8. Bypasses trtexec (Python API only). " "Experimental fix for Gemma RMSNorm NaN under INT8.") parser.add_argument("--force", action="store_true", help="Rebuild even if engine exists") parser.add_argument("--reset-timing-cache", action="store_true", help="Delete existing timing.cache before build. Use when switching " "from single-profile to multi-profile builds: the cached kernel " "selection for profile 0 shape from a single-profile build is " "suboptimal (693 emb/s) vs a fresh multi-profile build (3800 emb/s).") parser.add_argument("--onnx", default=None, help="Override ONNX path (bypasses find_onnx priority order)") parser.add_argument( "--role", default="embedding", choices=["embedding", "ner", "reranker"], help="Model role — selects optimization profile set. " "ner: batch≤32, seq≤128 (doc chunks). " "embedding: batch≤256, seq≤128 (default). " "reranker: same as embedding.", ) parser.add_argument("--model-id", default=None, help="Model ID for engine filename (default: basename of --model-dir)") parser.add_argument("--os-arch", default=None, help="OS+arch tag for engine filename (win_x64/linux_x64/linux_arm64/mac_arm64). Auto-detected if omitted.") parser.add_argument("--variant", default=None, help="Length-aware bucket variant name (embed: s/m/l; relation: s/m/l/xl). " "When set, collapses PROFILES to a rectangular (max_batch, max_seq) and " "appends `-{variant}` to the engine filename. The shape is taken from " "--variant-batch/--variant-seq when provided (config-driven, per-model " "from semantic_config.bucket_variants), else falls back to the embed-only " "BUCKET_VARIANT_PROFILES table. Enables cross-worker bucketed routing.") parser.add_argument("--variant-batch", type=int, default=None, help="Explicit rectangular batch for --variant (overrides BUCKET_VARIANT_PROFILES).") parser.add_argument("--variant-seq", type=int, default=None, help="Explicit rectangular seq for --variant (overrides BUCKET_VARIANT_PROFILES).") parser.add_argument("--no-dynseq", action="store_true", help="Bake a rectangular sequence dimension instead of the " "bucket's dynamic range (BUCKET_VARIANT_DYNSEQ). Costs " "about 10%% of embedding throughput; for A/B controls.") args = parser.parse_args() # Select profile set based on model role before any build function is called. # build_with_trtexec and build_with_python_api both read the global PROFILES. global PROFILES, DYNSEQ_MIN_SEQ if args.role == "ner": PROFILES = NER_PROFILES elif args.role == "reranker": PROFILES = RERANKER_PROFILES # --variant takes precedence over the role-default PROFILES. Bucketed variants # apply to embedding AND reranker (relation-CE length bucketing, 2026-07-10); # rejected for `--role ner` (NER is single-shape). The shape is config-driven: # --variant-batch/--variant-seq (forwarded by models/compile.cmd from the model's # semantic_config.bucket_variants) win; else fall back to the embed-only table. if args.variant is not None: if args.role not in ("embedding", "reranker"): print( f"ERROR: --variant {args.variant!r} is incompatible with --role {args.role!r}; " f"bucketed variants are for embedding/reranker only.", file=sys.stderr, ) sys.exit(1) if args.variant_batch is not None and args.variant_seq is not None: _vb, _vs = args.variant_batch, args.variant_seq elif args.variant in BUCKET_VARIANT_PROFILES: _vb, _vs = BUCKET_VARIANT_PROFILES[args.variant] else: print( f"ERROR: --variant {args.variant!r} has no shape: pass --variant-batch/--variant-seq " f"or use a known embed variant {list(BUCKET_VARIANT_PROFILES.keys())}.", file=sys.stderr, ) sys.exit(1) PROFILES = [(_vb, _vs, _vb, _vs)] print(f"[build_trt] --variant {args.variant} → rectangular ({_vb}, {_vs})", flush=True) # Dynamic sequence for the embed grid. Keyed on the SHAPE, not the bucket # name: relation-CE reuses s/m/l/xl at seq 64/128/256/320 and its opt # points are not these. A shape that does not match the embed table # stays rectangular. _dyn = BUCKET_VARIANT_DYNSEQ.get(args.variant) if _dyn and not args.no_dynseq and \ BUCKET_VARIANT_PROFILES.get(args.variant) == (_vb, _vs): DYNSEQ_MIN_SEQ, _opt = _dyn PROFILES = [(_vb, _vs, _vb, _opt)] print(f"[build_trt] --variant {args.variant} → dynamic seq " f"{DYNSEQ_MIN_SEQ}→{_opt}→{_vs} at batch {_vb}", flush=True) # Dynamic-seq probe — applied last so it wins over both the rectangular # --variant shape and the BUCKET_VARIANT_DYNSEQ table. This is the knob for # sweeping a candidate grid; production builds get their shape from the # table and leave it unset. _dynseq = os.environ.get("DYNSEQ_OVERRIDE", "").strip() if _dynseq: try: _db, _dmin, _dopt, _dmax = (int(x) for x in _dynseq.split(",")) PROFILES = [(_db, _dmax, _db, _dopt)] DYNSEQ_MIN_SEQ = _dmin print( f"[build_trt] DYNSEQ probe: batch {_db} rectangular, seq {_dmin}→{_dopt}→{_dmax}", flush=True, ) except (ValueError, AttributeError) as _e: print(f"[build_trt] WARN: bad DYNSEQ_OVERRIDE='{_dynseq}': {_e}", flush=True) use_bf16 = args.bf16 use_fp8 = args.fp8 and not use_bf16 use_int8 = args.int8 and not args.no_int8 and not use_fp8 and not use_bf16 use_fp16 = args.fp16 and not use_bf16 use_per_layer_norm = args.per_layer_norm and use_int8 and not use_bf16 use_strongly_typed = False cuda_arch = args.cuda_arch or get_cuda_arch() if not cuda_arch: print("ERROR: Cannot detect CUDA arch. Pass --cuda-arch sm_XXX", file=sys.stderr) sys.exit(1) # `--cuda-arch` names the FILE; TensorRT always builds for the GPU that is # present. So the detected architecture is the ONLY one this machine may # produce, and a request for any other is refused rather than honoured with # a rename. There is no override: an engine carrying an architecture it was # not built for fails `deserializeEngine` on the real card with no fallback, # and it fails there, on a user's machine, not here. Every other arch is # built on a host that has it (Selectel VMs for sm_75/86/89). # # This is not hypothetical: dist/ held 30 engines across four models named # sm_75/sm_86/sm_89 that were byte-identical to their sm_120 twins. _detected = get_cuda_arch() if _detected and args.cuda_arch and args.cuda_arch != _detected: print(f"ERROR: --cuda-arch {args.cuda_arch} but this machine is " f"{_detected}. TensorRT cannot build for an absent architecture; " f"the engine would be a mislabelled {_detected} plan. Build it on " f"a {args.cuda_arch} host.", file=sys.stderr) sys.exit(2) if args.onnx: onnx_path = args.onnx if not os.path.exists(onnx_path): print(f"ERROR: --onnx path not found: {onnx_path}", file=sys.stderr) sys.exit(1) else: # Mode-aware ONNX discovery: FP8 build → prefer model_fp8qdt.onnx; # INT8 → model_int8qdt.onnx; BF16 → plain model.onnx. # Without this, --fp8 mode would pick up model_int8qdt.onnx whose # Q/DQ nodes are INT8-typed, defeating the FP8 builder flag and # silently producing an INT8 engine. _discover_mode = "fp8" if use_fp8 else ("bf16" if use_bf16 else "int8") onnx_path = find_onnx(args.model_dir, mode=_discover_mode) if onnx_path is not None and use_fp8 and not onnx_is_fp8(onnx_path): # FP8 requested but no model_fp8qdt.onnx — find_onnx fell back # to INT8 Q/DQ ONNX. Per line 1124 logic this auto-enables # both --int8 and --fp8 builder flags, but TRT will only run # INT8 (no FP8 Q/DQ nodes in the file). The resulting engine # is INT8 disguised as FP8. Warn loudly so the user can run # `quantize_onnx.py --fp8` first. print(f"WARNING: --fp8 requested but {os.path.basename(onnx_path)} is INT8 Q/DQ.") print(f"WARNING: Resulting engine will be INT8 (no FP8 Tensor Cores).") print(f"WARNING: Run `scripts/quantize_onnx.py --fp8 --model-dir {args.model_dir}` first.") if not onnx_path: print(f"ERROR: ONNX model not found in {args.model_dir}", file=sys.stderr) sys.exit(1) # ⭐ MAKE THE FP16 LANE ACTUALLY FP16. # # TRT 11 is STRONGLY-TYPED ONLY: the per-precision builder flags were removed and # precision is read entirely from the ONNX tensor types. `catalog.py` routes # rerankers and `trt_fp16_only` models here with `--no-int8` and labels the build # FP16, but nothing converted the graph — so every engine on this lane shipped as # FP32. Measured 2026-08-03 across the deployed set: relation-v1 Float=335 Half=0, # reranker-v1 Float=325 Half=0, denoise-v1 Float=331 Half=0 (its catalog entry even # says BF16). Only embed-v1 was genuinely quantized, because its ONNX carries Q/DQ # nodes and strong typing therefore has something to read. # # Converting and rebuilding: relation 20.74 -> 5.44 ms/chunk (3.81x), reranker # 21.14 -> 5.40 (3.91x), denoise 8.56 -> 3.60 (2.38x), each verified against the # previously deployed engine at cos >= 0.999995 with argmax fully preserved. # Engines halve on disk too. It is ~4x rather than the ~1.4x a DRAM argument # predicts because FP32 matmuls do not use the tensor cores at all. # # The tag drives it, as it always did — this only makes the flag mean its label. # Re-verify a converted model with: # models/_compile/tools/build_fp16_engine.py --stage verify --model if not use_int8 and not use_fp8 and not use_bf16: try: sys.path.insert(0, os.path.join( os.path.dirname(os.path.dirname(os.path.dirname( os.path.abspath(__file__)))), "tools")) from onnx_fp16 import ensure_fp16_onnx onnx_path = ensure_fp16_onnx(onnx_path) except Exception as e: # noqa: BLE001 - never block a build on the conversion print(f"WARNING: FP16 ONNX conversion failed ({e}); " f"building FP32 from {os.path.basename(onnx_path)}") model_id = args.model_id or os.path.basename(os.path.normpath(args.model_dir)) if args.os_arch: os_arch = args.os_arch else: import platform as _platform _machine = _platform.machine().lower() _is_arm = _machine in ("arm64", "aarch64") if sys.platform == "win32": os_arch = "win_arm64" if _is_arm else "win_x64" elif sys.platform == "darwin": os_arch = "mac_arm64" else: os_arch = "linux_arm64" if _is_arm else "linux_x64" # Length-aware variant suffix slotted into filename BEFORE the runtime # tag — keeps the established `..._{os_arch}_trt{VER}_{cuda_arch}.engine` # cluster intact so existing CDN tooling continues to glob by suffix. # Layout: `{model_id}-{variant}_{os_arch}_trt{VER}_{cuda_arch}.engine`. variant_tag = f"-{args.variant}" if args.variant else "" rt_tag = _trt_runtime_tag() _refuse_trt_version_skew(rt_tag) engine_path = os.path.join(args.model_dir, f"{model_id}{variant_tag}_{os_arch}_{rt_tag}_{cuda_arch}.engine") if os.path.exists(engine_path) and not args.force: size_mb = os.path.getsize(engine_path) / (1024 * 1024) print(f"Engine already exists: {engine_path} ({size_mb:.1f} MB)") print("Use --force to rebuild") sys.exit(0) if args.reset_timing_cache: timing_cache_path = os.path.join(args.model_dir, "timing.cache") if os.path.exists(timing_cache_path): os.remove(timing_cache_path) print(f" Timing cache deleted: {timing_cache_path} (fresh kernel selection)") # Explicit Q/DQ ONNX detected: override precision mode based on Q/DQ flavour. # INT8 Q/DQ path → STRONGLY_TYPED: prevents TRT's backward type-inference from propagating # INT8 from DequantizeLinear's input to the shared Q/DQ scale initializer. # ORT's quantize_static creates one scale initializer shared by both Q and DQ in each pair. # In weakly-typed mode this causes "[SCALE] has invalid precision Int8, ignored" and disables # W8A16 fusion (verified 363 warnings = 363 DQ nodes, TRT 10.16, sm_120). # With STRONGLY_TYPED, TRT reads precision from ONNX data_type; no builder flags needed. # # FP8 Q/DQ path → also STRONGLY_TYPED (corrected 2026-05-19): the prior # "FP8 STRONGLY_TYPED = 171 emb/s" datum was measured on a plain `model.onnx` # WITHOUT Q/DQ nodes — TRT fell through to the generic codepath. With a # real FP8 Q/DQ ONNX from ModelOpt, STRONGLY_TYPED reads FP8 precision + # BF16 scale initializers directly from the graph, avoiding the # weakly-typed validate-error trap: `bf16 precision has been set ... but # bf16 is not configured in the builder` (TRT 10.16 silently rejects # `config.set_flag(BF16)` when also FP8+INT8+FP16 are set, sm_120). # No precision builder flags are set in this mode — TRT honours the # ONNX types as-is. # # INT8 Q/DQ ONNX + --fp8 (legacy mixed path) → keep weakly-typed # FP16+INT8+FP8: meaningful only when caller mixes an INT8 Q/DQ ONNX with # --fp8 hint. Currently no production model hits this branch; preserved # for backward compatibility. if onnx_has_qdq(onnx_path): use_bf16 = False use_per_layer_norm = False if onnx_is_fp8(onnx_path): # FP8 Q/DQ ONNX — STRONGLY_TYPED reads everything from graph. use_fp16 = False use_int8 = False use_fp8 = False # builder flags suppressed; FP8 comes from Q/DQ nodes use_strongly_typed = True print(f"[PTQ] Explicit FP8 Q/DQ ONNX → STRONGLY_TYPED (FP8 Tensor Cores + BF16 scales from ONNX)") elif use_fp8: # INT8 Q/DQ ONNX + --fp8: weakly-typed mixed path (legacy). use_fp16 = True use_int8 = True use_strongly_typed = False print(f"[PTQ] INT8 Q/DQ ONNX + --fp8 → FP16+INT8+FP8 (kFP8 Tensor Cores for Q/DQ layers)") else: # INT8 Q/DQ ONNX, current production path. use_fp16 = False use_int8 = False use_strongly_typed = True print(f"[PTQ] Explicit Q/DQ ONNX — STRONGLY_TYPED (W8A16 path, no [SCALE] type-inference warnings)") if use_strongly_typed: # Differentiate between INT8 Q/DQ (W8A16) and FP8 Q/DQ (E4M3+BF16). # `onnx_is_fp8` is filename-based and was already evaluated before # the flag-resolution block above. precision = "STRONGLY_TYPED+FP8" if onnx_is_fp8(onnx_path) else "STRONGLY_TYPED+W8A16" elif use_bf16: precision = "BF16" elif use_fp8: precision = "FP8+FP16" elif use_int8 and use_per_layer_norm: precision = "INT8+BF16-norm" elif use_int8: precision = "FP16+INT8" else: precision = "FP16" print(f"Building TRT engine: {cuda_arch} ({precision})") print(f" ONNX: {onnx_path}") print(f" Engine: {engine_path}") print() ok = False # Per-layer-norm mode requires Python API (trtexec has no per-layer precision CLI). # FP8 (weakly-typed kFP8+kFP16) and all other modes can use trtexec. trtexec = None if not use_per_layer_norm: trtexec = shutil.which("trtexec") if not trtexec: for p in [ "/usr/bin/trtexec", "/usr/local/bin/trtexec", ]: if os.path.isfile(p) and os.access(p, os.X_OK): trtexec = p break if not trtexec: import glob for pattern in [ "/usr/local/TensorRT-*/bin/trtexec", "/opt/tensorrt/bin/trtexec", "/usr/local/cuda/bin/trtexec", ]: matches = glob.glob(pattern) if matches: trtexec = matches[-1] # latest version break if trtexec: print(f"Using trtexec: {trtexec}") ok = build_with_trtexec(onnx_path, engine_path, trtexec_bin=trtexec) else: print("trtexec not in PATH — using Python TensorRT API") ok = build_with_python_api(onnx_path, engine_path) # TRT 11 has a SINGLE build path: strongly-typed, precision read from the # ONNX Q/DQ nodes. The TRT-10 fallback cascade (retry as weakly-typed # FP16+INT8 / BF16 / FP8 / calibrated-INT8) was removed along with those # builder flags + the calibrator — there is nothing weaker to fall back to. # If the strongly-typed build fails, it fails; the caller surfaces the TRT # error from the log above. if not ok or not os.path.exists(engine_path): print("ERROR: Engine build failed", file=sys.stderr) sys.exit(1) size_mb = os.path.getsize(engine_path) / (1024 * 1024) print(f"\nSUCCESS: {engine_path} ({size_mb:.1f} MB)") if __name__ == "__main__": main()