Download _build-tools/build_trt_engine.py from faxenoff/code-daemon-embed-v1: direct link, hf CLI and curl.
- Browser
- Download file 60.5 kB
-
https://huggingface.co/faxenoff/code-daemon-embed-v1/resolve/main/_build-tools/build_trt_engine.py
- Command line
-
hf download hf://faxenoff/code-daemon-embed-v1/_build-tools/build_trt_engine.py
-
curl -L -o build_trt_engine.py https://huggingface.co/faxenoff/code-daemon-embed-v1/resolve/main/_build-tools/build_trt_engine.py
60.5 kB
#!/usr/bin/env python3
"""Build TensorRT engine from ONNX model with multi-profile bucketing.
Usage:
python scripts/build_trt_engine.py [--model-dir PATH] [--cuda-arch sm_120]
Two modes:
1. trtexec (if in PATH) β NVIDIA's official CLI tool
2. Python TensorRT API (fallback) β uses tensorrt pip package
Profiles (3 buckets matching tensorrt_runtime.zig default_profiles):
0: batch 1-32, seq 1-32 (opt: 16x32) β short entity names
1: batch 1-64, seq 1-64 (opt: 32x64) β entity descriptions
2: batch 1-128, seq 1-128 (opt: 128x128) β docstrings / function bodies
Output: {model_id}_{os_arch}_trt{VER}_{cuda_arch}.engine in the model directory
(os_arch = win_x64 / linux_x64 / linux_arm64 / mac_arm64;
VER = TensorRT major.minor from vendor/tensorrt/VERSION, e.g. trt11.0).
A serialized plan is keyed on {GPU arch x OS x TRT version} and TensorRT offers
no compat fallback on deserialize, so the version belongs in the filename next
to the arch: the daemon then resolves an engine it can actually load instead of
opening a same-named plan from another TRT and failing inside the runtime.
"""
import argparse
import os
import shutil
import subprocess
import sys
import warnings
from pathlib import Path
# Force UTF-8 output on Windows (default console encoding is CP1251/CP1252)
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
if hasattr(sys.stderr, "reconfigure"):
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
# Auto-install pip deps on first run when this script is invoked directly
# (i.e. without going through models/code-daemon-embed/run.ps1 / run.sh which
# already install everything). Idempotent β skips packages that are already importable.
if os.environ.get("ULTRACODE_NO_BOOTSTRAP") != "1":
# Repo root by walking up to build.zig, not by counting `..` β this file has
# moved once already (scripts/ -> models/_compile/backends/workers/) and a
# hardcoded depth breaks silently, since a missing directory just means "no
# bootstrap available".
_root = Path(__file__).resolve()
for _ in range(6):
_root = _root.parent
if (_root / "build.zig").is_file():
break
_bootstrap_dir = _root / "models" / "code-daemon-embed"
if (_bootstrap_dir / "_bootstrap.py").exists():
sys.path.insert(0, str(_bootstrap_dir))
try:
from _bootstrap import ensure_deps, ensure_nvidia_deps
# onnx is needed for ensure_single_file_onnx() and proto inspection;
# torch is not required by this script β skip the torch install.
ensure_deps(extra=["onnx"], torch_required=False)
# tensorrt + cuda-python (NVIDIA-only). No-op on macOS / CPU hosts.
ensure_nvidia_deps()
except Exception as _bs_err:
print(f"[bootstrap] WARNING: {_bs_err}", flush=True)
finally:
sys.path.remove(str(_bootstrap_dir))
# Suppress TRT deprecation warnings: implicit INT8 calibration (superseded by QAT/explicit
# quantization in TRT 10.1+) and per-layer precision (superseded by strong typing in TRT 10.12+).
# Replacements require ONNX QAT nodes / STRONGLY_TYPED mode β not applicable here.
warnings.filterwarnings("ignore", message=".*Deprecated.*", category=DeprecationWarning)
warnings.filterwarnings("ignore", message=".*deprecated.*", category=DeprecationWarning)
def ensure_single_file_onnx(onnx_path):
"""Convert split ONNX (model.onnx + model.onnx_data) to single-file if needed."""
data_file = onnx_path + "_data"
if not os.path.exists(data_file):
return onnx_path
merged_path = onnx_path.replace(".onnx", "_merged.onnx")
if os.path.exists(merged_path):
merged_size = os.path.getsize(merged_path)
if merged_size < 1024 or merged_size > 2_000_000_000:
# Empty/corrupted or >2GB (protobuf can't parse) β delete stale cache
os.remove(merged_path)
print(f" Removed stale merged ONNX ({merged_size / 1e6:.0f} MB)")
else:
merged_mtime = os.path.getmtime(merged_path)
if (merged_mtime > os.path.getmtime(onnx_path) and
merged_mtime > os.path.getmtime(data_file)):
print(f" Split ONNX: using cached merged file")
return merged_path
try:
import onnx
print(f" Split ONNX detected: merging into single file...")
model = onnx.load(onnx_path, load_external_data=True)
onnx.save(model, merged_path)
merged_size = os.path.getsize(merged_path)
if merged_size > 2_000_000_000:
# >2GB: protobuf can't parse it back, trtexec may fail. Use original + external data.
os.remove(merged_path)
print(f" Merged {merged_size / 1e6:.0f} MB > 2GB limit, using split ONNX (parse_from_file handles external data)")
return onnx_path
print(f" Merged: {merged_size / (1024*1024):.1f} MB")
return merged_path
except Exception as e:
print(f" Split ONNX merge failed: {e}")
return onnx_path
# Profile specs: (max_batch, max_seq, opt_batch, opt_seq)
#
# Single rectangular profile per model type β no multi-profile bucketing.
# The daemon accumulates up to batch=128 before dispatch; buildEmbedInput
# caps at 420 chars β 64 tokens so seq=64 covers all real traffic.
# Fewer profiles = less VRAM, faster compile, trivial selectProfile (always 0).
# Multi-profile builds were tried and produce slower kernels for the seq=64 path
# on Gemma sm_120 (autotuner picks kernels optimized for the broader shape range).
#
# Profile specs: (max_batch, max_seq, opt_batch, opt_seq).
#
# 2026-05-05 β half-batch-zero saga resolved. The "TRT sm_120 regression"
# we chased turned out to be a Zig-side dtype mismatch: engine input
# binding is INT64 (verified via getTensorDtype), but our worker copied
# host buffers as INT32 stride. Engine read 8 bytes per token from a
# buffer where we'd written 4 β got 2Γ rows of garbage from uninit
# memory, manifesting as "second half of every batch is zero". Fixed
# in tensorrt_runtime.zig + embed_worker.zig (i32 β i64 throughout the
# TRT input chain). Engine itself is fine; restore full 128 batch.
PROFILES = [
(128, 64, 128, 64), # 0: rectangular fast path β batch 128, seq 64
]
RECTANGULAR_PROFILE = True
# Dynamic-seq probe (2026-07-28). Set `DYNSEQ_OVERRIDE="batch,min_seq,opt_seq,max_seq"`
# to build ONE engine whose SEQUENCE dimension is a range while the batch stays
# rectangular. Purpose: a chunk could then be dispatched at its own longest text
# instead of the bucket's ceiling β live telemetry says 15.3% of GPU token-work
# is padding (avg 42/48, 56/64, 70/80, 95/128). The question this answers is
# whether TRT's broader-range kernel choice costs more than the padding it saves;
# the file header already records that multi-PROFILE builds regressed seq=64 on
# sm_120. Applied AFTER --variant in main() so the engine still gets a suffixed
# filename. Production builds leave it unset.
DYNSEQ_MIN_SEQ = None
# βββ Experimental switches (hardcoded) ββββββββββββββββββββββββββββββββββ
# Project preference: in-source const flip is cleaner than env-var dance
# for one-off experiments. Each switch documents its evidence + revert.
# Flip the value, save, run models/compile or build_trt_engine directly.
# Builder workspace pool. Default 4 GB; bump to 8-12 GB to widen TRT
# tactic search on autotuning. Build-time only β does NOT affect engine
# size or runtime VRAM. Risk: build OOM if value > free VRAM at build time.
# REVERT: change back to 4.
TRT_WORKSPACE_GB = 8
# TRT max_aux_streams config.
# 0 β workaround for the 2026-04-28 "5 valid + N zeros" sm_120 codegen
# bug (TRT 10.13.3). Forces single-stream kernel generation.
# 4 β TRT default. Researcher confirmed 2026-05-19 that post-fusion
# embedding graphs (BGE/Qwen3/Gemma) are too linear for aux streams
# to help, and TRT 10.16 gracefully self-disables ("Multi stream is
# disabled because the stream assignment failed" warning). Either
# value produces functionally identical engines on this family.
# Kept as probe knob β REVERT: 0.
TRT_MAX_AUX_STREAMS = 0
# Env override for sweep experiments. Format: RECT_PROFILE_OVERRIDE="batch,seq"
# β collapses PROFILES to a single (batch, seq, batch, seq) entry. Production
# builds leave this unset. Example: RECT_PROFILE_OVERRIDE="32,32".
_rect_override = os.environ.get("RECT_PROFILE_OVERRIDE", "").strip()
if _rect_override:
try:
_b, _s = (int(x) for x in _rect_override.split(","))
PROFILES = [(_b, _s, _b, _s)]
print(f"[build_trt] PROFILES overridden via env: rectangular ({_b}, {_s})", flush=True)
except (ValueError, AttributeError) as _e:
print(f"[build_trt] WARN: bad RECT_PROFILE_OVERRIDE='{_rect_override}': {_e}", flush=True)
# Length-aware bucket variants β when `--variant {s,m,l}` is passed, PROFILES
# collapses to that variant's rectangular (max_batch, max_seq). Single source
# of truth lives in `models/code-daemon-embed/embed_bucket_config.py`. We
# duplicate the tuples here to avoid forcing the build to import from a model
# subdirectory at parse-time (the bootstrap path may not yet be on sys.path).
# `models/_compile/preflight_bucket_sync.py` runs as a pre-build gate to catch drift.
BUCKET_VARIANT_PROFILES = {
# 2026-07-27: grid 40/64/128 β 32/48/80/128, measured on real engines
# (see the header comment in src/inference/embed_bucket_config.zig).
"s": (96, 48),
"m": (128, 64),
"l": (128, 80),
"xl": (256, 128),
}
# (min_seq, opt_seq) per embed bucket β the SEQUENCE dimension is a range while
# the batch stays rectangular, so a chunk is dispatched at its own longest text
# instead of the bucket ceiling. Live telemetry put 15.3% of GPU token-work in
# sequence padding; recovering it measured +10.2% end to end (9,738 -> 10,731
# emb/s on mysql-server) and is quality-neutral by construction β padding is
# masked in attention, verified at cosine 1.000000 on 128 rows of real ids.
#
# Two rules produced these numbers and neither is guessable:
# - `opt` sits at the bucket's histogram-weighted MEAN, not its ceiling. The
# tax is paid for being far from the opt point, not for being dynamic: one
# wide 16β64β128 engine costs +0.8% at seq 64 and +15.4% at seq 80.
# - `min` is the bucket FLOOR, not the observed corpus minimum β `s` gets 8
# rather than the measured 24, because the router must accept a 3-token text.
#
# This lived in a DYNSEQ_OVERRIDE env var and was deployed by copying files over
# the rectangular ones. A later `models/compile` run rebuilt them rectangular and
# the +10.2% was silently gone; the engines still loaded, still returned 768-dim
# vectors, and reported 54 layers instead of 69 where nobody was looking. Hence
# the table: a plain build now produces the shape that is meant to ship.
BUCKET_VARIANT_DYNSEQ = {
"s": (8, 48),
"m": (56, 56),
"l": (72, 72),
"xl": (88, 96),
}
# NER single profile: batch=32 (NER worker --batch-size 32), seq=128.
# Bumped from seq=64 (2026-05-05) β ~20% of incoming chunks exceed 64
# tokens (long entity descriptions, function bodies, docstrings), causing
# `SetShapeFailed` at runtime and DocGraph falling back to rules-only for
# those entities. The OV NER variants ship as `_b32_s128`, so the ONNX
# already supports seq=128. Opt point sits at 96 β middle of the 1..128
# range, biasing the autotuner toward the typical mass of inputs without
# regressing the long tail.
NER_PROFILES = [
# 0: NER main path β RECTANGULAR (max == opt) to avoid an invalid
# min<=opt<=max profile. The previous value (32, 128, 32, 96) set
# opt_seq=96 < max_seq=128, which under RECTANGULAR_PROFILE (min==max==128)
# makes opt_seq < min_seq β TRT 11 rejects with "Invalid optimization
# profiles ... ensure MIN<=OPT<=MAX". The NER worker always pads to
# batch=32 seq=128, so a static 32x128 profile is correct.
(32, 128, 32, 128),
]
# Reranker single profile: batch=16 (reranker worker max_batch=16, matches the OV engine
# `_b16_s256` and models/compile.cmd's $ovChkBatch=16), seq=256. The cross-encoder scores
# (query, document) PAIRS, which are long, so seq must be 256 β NOT the embedding default
# (128, 64). Without a reranker-specific profile, `--role reranker` fell through to the
# embedding PROFILES (128, 64); the engine bound seq=64 while handleRerank pads to
# max_seq_len=256 β SetShapeFailed at runtime β search degraded to no-rerank. RECTANGULAR
# (min==opt==max) per handleRerank's rectangular-engine contract (embed_worker.zig:4867).
RERANKER_PROFILES = [
(16, 256, 16, 256),
]
def _installed_trt_tag():
"""`trt{major}.{minor}` of the TensorRT in THIS interpreter, or None."""
try:
import tensorrt as trt
parts = str(trt.__version__).split(".")
except Exception:
return None
return f"trt{parts[0]}.{parts[1]}" if len(parts) >= 2 and parts[0] else None
def _pinned_trt_tag():
"""`trt{major}.{minor}` from `vendor/tensorrt/VERSION`, or None if no repo.
This is the pin build.zig turns into the daemon's `runtime_versions.trt_tag`,
so naming from it guarantees the engine we write is the engine the worker
looks for. Absent when the script runs standalone β the cloud sm_86/sm_89
builder (models/_cloud_shared/build-tools/build_engines.sh) copies only this
.py onto a bare VM, with no vendor/ tree to read.
"""
d = os.path.dirname(os.path.abspath(__file__))
for _ in range(6): # walk up to repo root
pin = os.path.join(d, "vendor", "tensorrt", "VERSION")
if os.path.isfile(pin):
try:
parts = open(pin).read().strip().split(".")
except OSError:
return None
return f"trt{parts[0]}.{parts[1]}" if len(parts) >= 2 and parts[0] else None
parent = os.path.dirname(d)
if parent == d:
return None
d = parent
return None
def _trt_runtime_tag():
"""Runtime-version tag embedded in every engine filename, e.g. `trt11.0`.
Prefers the repo pin (what the DAEMON will deserialize with); falls back to
the TensorRT actually doing the build, so a standalone/cloud run still
produces a correctly-labelled engine instead of failing. Only when neither is
knowable do we stop: an untagged engine is invisible to the daemon, and TRT
builds are far too slow to discover that afterwards.
"""
tag = _pinned_trt_tag() or _installed_trt_tag()
if tag:
return tag
print("ERROR: cannot determine the TensorRT version β neither vendor/tensorrt/VERSION")
print(" nor an importable `tensorrt` package. Engine filenames carry the")
print(" TensorRT major.minor and the daemon only loads its own version.")
sys.exit(1)
def _refuse_trt_version_skew(rt_tag):
"""Stop when the TensorRT building this engine is not the pinned one.
Matters more here than anywhere else: the plan is produced by the `tensorrt`
package in THIS interpreter but deserialized by the vendored nvinfer the
daemon links. If they differ, the file gets a name promising a runtime that
cannot read it β a load failure the version tag exists to prevent.
This was a WARNING until 2026-09-12. It printed on every Linux build for
weeks while every Linux engine was built by 11.2 and named trt11.0
(`models/_compile/deps/wsl.py:_trt_pin`), because nothing consumed it. There
is no legitimate skew: a patch-level difference yields the same tag and never
reaches this branch, and a standalone cloud run has no pin, so `rt_tag` IS the
installed tag there. A missing package is handled elsewhere.
"""
installed_tag = _installed_trt_tag()
if installed_tag and installed_tag != rt_tag:
print(f"ERROR: TensorRT version skew β this interpreter has {installed_tag} "
f"but the engine would be named {rt_tag} (from vendor/tensorrt/VERSION).")
print(f" The daemon deserializes with the pinned runtime and would "
f"REJECT the plan. Install tensorrt-cu13 at the pin, or bump the pin.")
sys.exit(1)
def get_cuda_arch():
"""Detect CUDA compute capability via nvidia-smi."""
try:
out = subprocess.check_output(
["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"],
text=True,
).strip().split("\n")[0]
major, minor = out.split(".")
return f"sm_{major}{minor}"
except Exception:
return None
def _onnx_is_valid(path):
"""Quick-check that an ONNX file is parseable (not truncated).
Files >1 MB that fail onnx.load are assumed valid: Python protobuf has a 2 GB
parse limit but trtexec uses the C++ ONNX parser which has no such limit."""
try:
import onnx
onnx.load(path, load_external_data=False)
return True
except Exception as e:
try:
size = os.path.getsize(path)
if size > 1024 * 1024:
print(f" NOTE: onnx.load failed ({e}), file is {size / (1024*1024):.0f} MB "
f"β assuming valid (trtexec uses C++ parser, no 2 GB limit)")
return True
except Exception:
pass
return False
def _has_all_external_data(onnx_path):
"""Return True if every external-data file referenced by the ONNX header exists
in the same directory. Returns True (optimistic) when the header itself cannot
be loaded (e.g. 2 GB+ merged file β no external refs to check)."""
try:
import onnx as _onnx
header = _onnx.load(onnx_path, load_external_data=False)
except Exception:
return True # can't check β assume ok (2 GB+ merged, no external refs)
onnx_dir = os.path.dirname(os.path.abspath(onnx_path))
for t in header.graph.initializer:
if t.data_location == _onnx.TensorProto.EXTERNAL:
for entry in t.external_data:
if entry.key == "location":
if not os.path.exists(os.path.join(onnx_dir, entry.value)):
return False
return True
def find_onnx(model_dir, mode="int8"):
"""Find ONNX model in model directory, with validation and cache fallback.
`mode` controls preference order:
- "fp8": prefer `model_fp8qdt.onnx` (ModelOpt FP8 PTQ output) over INT8.
- "int8" (default): prefer `model_int8qdt.onnx` (ModelOpt INT8 PTQ output).
- "bf16": prefer plain `model.onnx`.
Falls back through the alternate-precision PTQ ONNX before raw `model.onnx`
so that a misconfigured catalog still produces a build (TRT will pick up
whatever Q/DQ nodes are in the file).
"""
# Last-resort fallback: the compiler's download cache for third-party models
# (`models/_compile/.cache/models/<id>/`). The compiler itself always passes
# `--onnx` explicitly and the cloud builder stages the ONNX into --model-dir,
# so this is only reached by a hand-run invocation.
#
# It used to be derived as `<this script's dir>/.cache/<id>`, which pointed
# at `scripts/.cache` while the script lived there and became a path that can
# never exist once it moved.
model_name = os.path.basename(os.path.normpath(model_dir))
_pkg_dir = os.path.dirname(os.path.abspath(__file__)) # β¦/backends/workers
_compile_root = os.path.dirname(os.path.dirname(_pkg_dir)) # β¦/models/_compile
cache_dir = os.environ.get("ULTRACODE_COMPILE_CACHE") or \
os.path.join(_compile_root, ".cache")
cache_path = os.path.join(cache_dir, "models", model_name, "model.onnx")
int8_candidates = [
os.path.join(model_dir, "onnx", "model_int8qdt.onnx"),
os.path.join(model_dir, "model_int8qdt.onnx"),
os.path.join(cache_dir, "models", model_name, "model_int8qdt.onnx"),
]
fp8_candidates = [
os.path.join(model_dir, "onnx", "model_fp8qdt.onnx"),
os.path.join(model_dir, "model_fp8qdt.onnx"),
os.path.join(cache_dir, "models", model_name, "model_fp8qdt.onnx"),
]
plain_candidates = [
os.path.join(model_dir, "model.onnx"),
os.path.join(model_dir, "onnx", "model.onnx"),
cache_path,
]
if mode == "fp8":
candidates = fp8_candidates + int8_candidates + plain_candidates
elif mode == "bf16":
candidates = plain_candidates + int8_candidates + fp8_candidates
else:
candidates = int8_candidates + plain_candidates + fp8_candidates
for p in candidates:
if os.path.exists(p):
if not _onnx_is_valid(p):
print(f" WARNING: {p} is corrupted (truncated?), trying next candidate")
continue
if not _has_all_external_data(p):
print(f" NOTE: {p} has missing external data files β trying next candidate")
continue
return p
return None
def onnx_has_qdq(onnx_path):
"""True if the ONNX model contains explicit Q/DQ nodes (PTQ quantized).
When Q/DQ nodes are present, TRT must have a matching precision builder
flag enabled (INT8 for int8qdt, FP8 for fp8qdt). Without it, TRT silently
ignores DequantizeLinear/QuantizeLinear and builds a plain BF16/FP16
engine, wasting the calibration work.
Detection by filename is sufficient β quantize_onnx.py always writes
'model_int8qdt.onnx' (INT8 mode) or 'model_fp8qdt.onnx' (FP8 mode).
"""
name = os.path.basename(onnx_path)
return any(tag in name for tag in ("int8qdt", "fp8qdt", "nvfp4qdt", "int4qdt"))
def onnx_is_fp8(onnx_path):
"""True if the ONNX model is a ModelOpt FP8 PTQ output (fp8qdt naming).
Used to distinguish FP8-quantized ONNX (needs --fp8 builder flag) from
INT8-quantized ONNX (needs --int8). When neither, fallback paths apply.
"""
return "fp8qdt" in os.path.basename(onnx_path)
def build_with_trtexec(onnx_path, engine_path, trtexec_bin="trtexec"):
"""Build engine using trtexec CLI (TRT 11: strongly-typed only)."""
onnx_path = ensure_single_file_onnx(onnx_path)
cmd = [trtexec_bin, f"--onnx={onnx_path}", f"--saveEngine={engine_path}"]
# TRT 11: precision is read entirely from the ONNX Q/DQ nodes. Strongly typed
# is the only typing mode β the per-precision flags (--int8/--fp16/--bf16/--fp8)
# and the INT8 calibrator (--calib) were removed. `--stronglyTyped` is not
# passed: TRT 11.3 release notes list it as a flag with no effect.
# Timing cache: speeds up rebuilds dramatically (minutes β seconds)
timing_cache = os.path.join(os.path.dirname(engine_path), "timing.cache")
cmd.append(f"--timingCacheFile={timing_cache}")
# Max builder optimization level for best engine quality
cmd.append("--builderOptimizationLevel=5")
# 2026-04-28 β sm_120 aux-stream regression fix (researcher rec #4 in
# .autodoc/archive/inference-backends.md). TRT 10.13.3 / 10.16.1 release notes
# document a multi-stream codegen bug on consumer Blackwell (sm_120)
# that affects pooling/attention subgraphs and produces partial
# outputs (the embeddinggemma "5 valid + N-5 zeros" symptom matches
# this signature: kernel runs but writes zeros to trailing rows).
# Disabling aux streams bypasses the buggy codegen path; cost is
# one stream of dispatch concurrency we don't currently exploit.
#
# 2026-05-19 β value lives at module top as TRT_MAX_AUX_STREAMS const.
# Flip there + rebuild to probe (researcher confirmed it's effectively
# a no-op on embedding models; TRT 10.16 self-disables anyway).
cmd.append(f"--maxAuxStreams={TRT_MAX_AUX_STREAMS}")
if TRT_MAX_AUX_STREAMS != 0:
print(f" [build_trt] TRT_MAX_AUX_STREAMS={TRT_MAX_AUX_STREAMS} β sm_120 workaround DISABLED")
# `--useCudaGraph` / `--useSpinWait` were passed here until 2026-09-22. They
# only shaped trtexec's post-build benchmark, never the saved engine, and
# TRT 11.3 release notes list both as flags with no effect β so a benchmark
# line printed by an 11.3 trtexec is measured WITHOUT graphs and does not
# compare with 10.16/11.0 logs. The runtime counterpart was measured and
# left off: `cuda_graphs_enabled` in src/inference/tensorrt_runtime.zig.
for max_b, max_s, opt_b, opt_s in PROFILES:
cmd.extend([
f"--minShapes=input_ids:1x1,attention_mask:1x1",
f"--optShapes=input_ids:{opt_b}x{opt_s},attention_mask:{opt_b}x{opt_s}",
f"--maxShapes=input_ids:{max_b}x{max_s},attention_mask:{max_b}x{max_s}",
])
print(f" trtexec command: {' '.join(cmd[:5])} ...")
result = subprocess.run(cmd)
return result.returncode == 0
def _add_nvinfer_to_path():
"""Add vendor/tensorrt/lib AND pip tensorrt_libs to DLL search path."""
added = []
# 1. vendor/tensorrt/lib (our bundled runtime DLLs).
# This file lives at models/_compile/backends/workers/, so the repo root is
# four levels up. It used to be one (`scripts/../vendor`), which stopped
# resolving the moment the script moved β silently, because the directory
# simply "was not there" and the pip tensorrt_libs below usually covered it.
_here = os.path.dirname(os.path.abspath(__file__))
_repo_root = os.path.abspath(os.path.join(_here, "..", "..", "..", ".."))
vendor_lib = os.path.join(_repo_root, "vendor", "tensorrt", "lib")
if os.path.isdir(vendor_lib):
added.append(vendor_lib)
# 2. pip tensorrt_libs (contains nvinfer_plugin_10.dll needed for builder)
try:
import site
for sp in site.getsitepackages() + [site.getusersitepackages()]:
trt_libs = os.path.join(sp, "tensorrt_libs")
if os.path.isdir(trt_libs):
added.append(trt_libs)
break
except Exception:
pass
# Register all paths
for lib_dir in added:
if sys.platform == "win32" and hasattr(os, "add_dll_directory"):
try:
os.add_dll_directory(lib_dir)
except OSError:
pass
os.environ["PATH"] = lib_dir + os.pathsep + os.environ.get("PATH", "")
return added[0] if added else None
def _import_tensorrt():
"""Import tensorrt, handling the cu12_bindings layout."""
_add_nvinfer_to_path()
# Try direct import first (tensorrt meta-package installed)
try:
import tensorrt
return tensorrt
except (ImportError, ModuleNotFoundError, FileNotFoundError, OSError):
pass
# Try bindings-only import (tensorrt_cu12_bindings installed)
try:
from tensorrt_bindings import tensorrt
return tensorrt
except (ImportError, ModuleNotFoundError, FileNotFoundError, OSError):
pass
return None
def _has_cuda_runtime():
"""Check if CUDA runtime is available (libcudart loadable)."""
import ctypes
for lib in ["libcudart.so", "libcudart.so.13", "libcudart.so.12", "libcudart.so.11",
"cudart64_13.dll", "cudart64_12.dll", "cudart64_110.dll"]:
try:
ctypes.cdll.LoadLibrary(lib)
return True
except OSError:
pass
# Also check if nvcc is in PATH (CUDA toolkit installed)
if shutil.which("nvcc"):
return True
return False
def ensure_tensorrt_package():
"""Auto-install tensorrt Python bindings + libs if missing."""
trt = _import_tensorrt()
if trt is not None:
return True
if not _has_cuda_runtime():
print(" tensorrt not available: CUDA runtime not found (no libcudart/nvcc)", file=sys.stderr)
print(" Install CUDA toolkit or add trtexec to PATH", file=sys.stderr)
return False
print(" tensorrt not found β installing via pip...")
nvidia_idx = "https://pypi.nvidia.com"
# Detect CUDA major version to pick matching tensorrt build. Mixing cu12
# and cu13 packages produces version-mismatch errors at engine deserialize
# because the engine plan encodes the build-time native DLL version, and
# `import tensorrt` then resolves to a different package's DLL at runtime.
cuda_major = None
try:
nvsmi = subprocess.check_output(
["nvidia-smi"], stderr=subprocess.DEVNULL, text=True, timeout=5,
)
import re
m = re.search(r"CUDA Version:\s*(\d+)\.(\d+)", nvsmi)
if m:
cuda_major = int(m.group(1))
except Exception:
pass
if cuda_major and cuda_major >= 13:
candidates = [
["tensorrt-cu13-bindings==11.0.0.114", "tensorrt-cu13-libs==11.0.0.114"],
["tensorrt-cu13-bindings", "tensorrt-cu13-libs"],
]
else:
candidates = [
["tensorrt-cu12-bindings==11.0.0.114", "tensorrt-cu12-libs==11.0.0.114"],
["tensorrt-cu12-bindings", "tensorrt-cu12-libs"],
]
for pkgs in candidates:
result = subprocess.run(
[sys.executable, "-m", "pip", "install", "--quiet",
"--only-binary", ":all:",
"--extra-index-url", nvidia_idx] + pkgs,
capture_output=True, text=True,
)
if result.returncode == 0:
break
else:
# Last resort: try the meta-package
subprocess.run(
[sys.executable, "-m", "pip", "install", "--quiet",
"--only-binary", ":all:",
"--extra-index-url", nvidia_idx,
"tensorrt"],
capture_output=True, text=True,
)
trt = _import_tensorrt()
if trt is not None:
print(f" tensorrt {trt.__version__} ready")
return True
print(" tensorrt still not importable after install", file=sys.stderr)
return False
def _upgrade_onnx_opset(onnx_path, target_opset=17):
"""Upgrade ONNX model to target opset if needed (fixes LayerNorm FP16 overflow)."""
# Handle split ONNX first β skip upgrade if merge fails (>2GB models)
data_file = onnx_path + "_data"
if os.path.exists(data_file):
merged = ensure_single_file_onnx(onnx_path)
if merged == onnx_path:
# Merge failed (protobuf 2GB limit) β skip opset upgrade, TRT will parse directly
print(f" Opset upgrade skipped: split ONNX too large to merge")
return onnx_path
onnx_path = merged
else:
onnx_path = ensure_single_file_onnx(onnx_path)
try:
import onnx
from onnx import version_converter
model = onnx.load(onnx_path)
current_opset = model.opset_import[0].version if model.opset_import else 0
if current_opset >= target_opset:
return onnx_path # already sufficient
print(f" Upgrading ONNX opset {current_opset} β {target_opset} (fixes LayerNorm FP16 overflow)")
model = version_converter.convert_version(model, target_opset)
upgraded_path = onnx_path.replace(".onnx", f"_opset{target_opset}.onnx")
onnx.save(model, upgraded_path)
return upgraded_path
except Exception as e:
print(f" ONNX opset upgrade skipped: {e}")
return onnx_path
def build_with_python_api(onnx_path, engine_path, fp16=True, int8=False, fp8=False, bf16=False, per_layer_norm=False, strongly_typed=False):
"""Build engine using TensorRT Python API (no trtexec needed)."""
if not ensure_tensorrt_package():
return False
trt = _import_tensorrt()
# Upgrade ONNX opset to 17+ for native INormalizationLayer (fixes FP16 LayerNorm overflow)
onnx_path = _upgrade_onnx_opset(onnx_path, target_opset=17)
logger = trt.Logger(trt.Logger.WARNING)
builder = trt.Builder(logger)
# TRT 11: EXPLICIT_BATCH was removed (explicit batch is the only mode) and
# STRONGLY_TYPED is the default + only typing mode. Precision comes entirely
# from the ONNX Q/DQ nodes β TRT reads each tensor's data_type as-is, with no
# backward type-inference that would propagate INT8 onto a DequantizeLinear's
# shared scale tensor. Per-precision BuilderFlags, the INT8 calibrator, and
# per-layer precision were ALL removed in TRT 11; this is now the only path.
network_flags = 1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED)
network = builder.create_network(network_flags)
print(" STRONGLY_TYPED: enabled (TRT 11 default; precision from Q/DQ ONNX nodes)")
parser = trt.OnnxParser(network, logger)
print(f" Parsing ONNX: {onnx_path}")
# Use parse_from_file if available (TRT 8.6+) β resolves external data (model.onnx_data)
# relative to the ONNX file directory. Falls back to parse(bytes) + chdir for older TRT.
parsed_ok = False
if hasattr(parser, "parse_from_file"):
parsed_ok = parser.parse_from_file(onnx_path)
else:
# Older TRT: chdir to model directory so external data is found
prev_cwd = os.getcwd()
os.chdir(os.path.dirname(os.path.abspath(onnx_path)))
try:
with open(onnx_path, "rb") as f:
parsed_ok = parser.parse(f.read())
finally:
os.chdir(prev_cwd)
if not parsed_ok:
for i in range(parser.num_errors):
print(f" ONNX parse error: {parser.get_error(i)}", file=sys.stderr)
return False
config = builder.create_builder_config()
# Workspace pool β bigger = wider tactic search. Const TRT_WORKSPACE_GB
# at module top. Default 4 GB; bumping to 8-12 GB widens tactic search.
# Build-time only β freed when builder exits.
config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, TRT_WORKSPACE_GB << 30)
print(f" Workspace pool: {TRT_WORKSPACE_GB} GB" + (" (default)" if TRT_WORKSPACE_GB == 4 else " (probe value)"))
# Max builder optimization level for best engine quality
config.builder_optimization_level = 5
# Detailed profiling info β needed for the engine_inspector dump below
# to enumerate fused layers (MHA, GEMM, SkipLayerNormalization, etc).
# Negligible build-time overhead; engine itself is unchanged.
try:
config.profiling_verbosity = trt.ProfilingVerbosity.DETAILED
except AttributeError:
pass
# 2026-04-28 β sm_120 aux-stream regression fix (researcher rec #4).
# TRT 10.13.3 / 10.16.1 release notes document a multi-stream
# codegen bug on consumer Blackwell (sm_120). Setting max_aux_streams
# to 0 bypasses the buggy path. Property added in TRT 8.6+; older
# versions silently ignore (try/except for safety).
#
# 2026-05-19 β value from module-top const TRT_MAX_AUX_STREAMS.
try:
config.max_aux_streams = TRT_MAX_AUX_STREAMS
if TRT_MAX_AUX_STREAMS == 0:
print(" max_aux_streams: 0 (sm_120 regression workaround)")
else:
print(f" max_aux_streams: {TRT_MAX_AUX_STREAMS} (probe value β TRT may auto-disable on linear graphs)")
except AttributeError:
pass
# Precision: TRT 11 reads every tensor type from the strongly-typed network
# (the ONNX Q/DQ nodes). The per-precision BuilderFlags (INT8/FP16/BF16/FP8),
# the INT8 calibrator (IInt8EntropyCalibrator2 + int8_calibrator +
# set_calibration_profile), and per-layer precision (layer.precision /
# set_output_type) were ALL removed in TRT 11 β there are no builder flags to
# set here. The INT8 W8A16 / FP8 E4M3 / NVFP4 path is selected purely by which
# *_qdt.onnx was produced by quantize_onnx.py.
print(" Precision: from ONNX Q/DQ nodes (strongly-typed; no builder flags)")
# Timing cache: speeds up rebuilds dramatically
timing_cache_path = os.path.join(os.path.dirname(engine_path), "timing.cache")
if os.path.exists(timing_cache_path):
with open(timing_cache_path, "rb") as f:
cache = config.create_timing_cache(f.read())
print(f" Timing cache: loaded ({os.path.getsize(timing_cache_path)} bytes)")
else:
cache = config.create_timing_cache(b"")
config.set_timing_cache(cache, ignore_mismatch=False)
# Create optimization profiles
for i, (max_b, max_s, opt_b, opt_s) in enumerate(PROFILES):
profile = builder.create_optimization_profile()
# RECTANGULAR_PROFILE: min == opt == max. TRT autotuner picks a
# single set of tactics for the exact (batch, seq) shape β no
# variable-batch search, no autotuner regression on sm_120.
# Trapezoid mode: min=(1,1) β opt β max. Lets TRT accept smaller
# real batches at runtime, but autotuner must cover the whole
# range and on consumer Blackwell that range produces broken
# kernels for the upper half of the batch dimension.
if DYNSEQ_MIN_SEQ is not None:
# Batch stays rectangular (the sm_120 batch-range codegen problem
# above is untouched); only seq becomes a range.
min_b, min_s = max_b, DYNSEQ_MIN_SEQ
elif RECTANGULAR_PROFILE:
min_b, min_s = max_b, max_s
else:
min_b, min_s = 1, 1
profile.set_shape("input_ids", (min_b, min_s), (opt_b, opt_s), (max_b, max_s))
profile.set_shape("attention_mask", (min_b, min_s), (opt_b, opt_s), (max_b, max_s))
# token_type_ids if present in the model
for j in range(network.num_inputs):
inp = network.get_input(j)
if inp.name == "token_type_ids":
profile.set_shape("token_type_ids", (min_b, min_s), (opt_b, opt_s), (max_b, max_s))
break
config.add_optimization_profile(profile)
shape_kind = "RECTANGULAR" if RECTANGULAR_PROFILE else "trapezoid"
print(f" Profile {i}: {shape_kind} {min_b}x{min_s} β {opt_b}x{opt_s} β {max_b}x{max_s}")
print(f" Building engine (this may take several minutes)...")
serialized = builder.build_serialized_network(network, config)
if serialized is None:
print("ERROR: Engine build failed", file=sys.stderr)
return False
os.makedirs(os.path.dirname(engine_path), exist_ok=True)
with open(engine_path, "wb") as f:
f.write(serialized)
# Save timing cache for future rebuilds
with open(timing_cache_path, "wb") as f:
f.write(cache.serialize())
print(f" Timing cache saved: {timing_cache_path}")
# Layer-info sidecar (for MHA fusion verification, autotuner tactic
# inspection). Deserialize the just-saved engine, dump full JSON of
# all layers via engine_inspector. Needs profiling_verbosity=DETAILED
# at build time (set above) for tactic names + dtypes to be present.
layers_json_path = engine_path.replace(".engine", ".layers.json")
try:
runtime = trt.Runtime(trt.Logger(trt.Logger.WARNING))
eng = runtime.deserialize_cuda_engine(serialized)
if eng is not None:
inspector = eng.create_engine_inspector()
layer_json = inspector.get_engine_information(trt.LayerInformationFormat.JSON)
with open(layers_json_path, "w", encoding="utf-8") as fj:
fj.write(layer_json)
n_layers = layer_json.count('"LayerType"')
n_int8 = layer_json.count('"Int8"')
n_fp8 = layer_json.count('"Fp8"') + layer_json.count('"FP8"')
n_bf16 = layer_json.count('"BFloat16"') + layer_json.count('"BF16"')
# Half and Float were missing from this verdict, so an all-FP32 engine and
# a fully-FP16 one printed the identical "Int8=0 BF16=0 FP8=0" line. That
# blind spot is how the FP16 lane shipped FP32 engines unnoticed β the one
# report that could have caught it did not look at the two types involved.
n_half = layer_json.count('"Half"')
n_float = layer_json.count('"Float"')
# TRT 11's Myelin names the fused kernel `_gemm_mha_v2_*`; the two older
# spellings never appear in a TRT 11 dump, so counting only them printed
# NOT DETECTED for every engine on 2026-09-11 - including all eleven
# XLM-R engines that carry 24 of these kernels each.
lj = layer_json.lower()
n_fmha = lj.count('fmha') + lj.count('fused_multi_head_attention') + lj.count('_gemm_mha')
print(f" Layer info: {layers_json_path}")
print(f" {n_layers} layers β Half={n_half} Float={n_float} "
f"Int8={n_int8} BF16={n_bf16} FP8={n_fp8}")
if n_half == 0 and n_int8 == 0 and n_bf16 == 0 and n_fp8 == 0 and n_float > 0:
print(" WARNING: engine is ENTIRELY FP32 β no reduced precision took "
"effect. TRT 11 reads precision from the ONNX types only.")
if n_fmha > 0:
print(f" MHA fusion: DETECTED ({n_fmha} fmha tactic refs)")
else:
print(f" MHA fusion: NOT DETECTED β consider ONNX-level attention export")
except Exception as e:
print(f" Layer-info dump SKIPPED: {e}")
return True
def main():
parser = argparse.ArgumentParser(description="Build TRT engine from ONNX")
parser.add_argument(
"--model-dir",
default="dist/models/multilingual-e5-small",
help="Path to model directory containing model.onnx",
)
parser.add_argument("--cuda-arch", default=None, help="CUDA arch (e.g. sm_120)")
parser.add_argument("--fp16", action="store_true", default=True, help="Enable FP16")
parser.add_argument("--int8", action="store_true", default=True, help="Enable INT8 mixed precision")
parser.add_argument("--no-int8", action="store_true", help="Disable INT8 (FP16 only)")
parser.add_argument("--fp8", action="store_true", help="Enable FP8 (E4M3) β Hopper/Blackwell only. Implies --no-int8.")
parser.add_argument("--bf16", action="store_true", help="Enable BF16 (FP32-range half-storage) β required for Gemma RMSNorm. Disables FP16/INT8/FP8.")
parser.add_argument("--per-layer-norm", action="store_true",
help="Per-layer precision: norm layers β BF16, rest β INT8. "
"Requires --int8. Bypasses trtexec (Python API only). "
"Experimental fix for Gemma RMSNorm NaN under INT8.")
parser.add_argument("--force", action="store_true", help="Rebuild even if engine exists")
parser.add_argument("--reset-timing-cache", action="store_true",
help="Delete existing timing.cache before build. Use when switching "
"from single-profile to multi-profile builds: the cached kernel "
"selection for profile 0 shape from a single-profile build is "
"suboptimal (693 emb/s) vs a fresh multi-profile build (3800 emb/s).")
parser.add_argument("--onnx", default=None, help="Override ONNX path (bypasses find_onnx priority order)")
parser.add_argument(
"--role", default="embedding", choices=["embedding", "ner", "reranker"],
help="Model role β selects optimization profile set. "
"ner: batchβ€32, seqβ€128 (doc chunks). "
"embedding: batchβ€256, seqβ€128 (default). "
"reranker: same as embedding.",
)
parser.add_argument("--model-id", default=None,
help="Model ID for engine filename (default: basename of --model-dir)")
parser.add_argument("--os-arch", default=None,
help="OS+arch tag for engine filename (win_x64/linux_x64/linux_arm64/mac_arm64). Auto-detected if omitted.")
parser.add_argument("--variant", default=None,
help="Length-aware bucket variant name (embed: s/m/l; relation: s/m/l/xl). "
"When set, collapses PROFILES to a rectangular (max_batch, max_seq) and "
"appends `-{variant}` to the engine filename. The shape is taken from "
"--variant-batch/--variant-seq when provided (config-driven, per-model "
"from semantic_config.bucket_variants), else falls back to the embed-only "
"BUCKET_VARIANT_PROFILES table. Enables cross-worker bucketed routing.")
parser.add_argument("--variant-batch", type=int, default=None,
help="Explicit rectangular batch for --variant (overrides BUCKET_VARIANT_PROFILES).")
parser.add_argument("--variant-seq", type=int, default=None,
help="Explicit rectangular seq for --variant (overrides BUCKET_VARIANT_PROFILES).")
parser.add_argument("--no-dynseq", action="store_true",
help="Bake a rectangular sequence dimension instead of the "
"bucket's dynamic range (BUCKET_VARIANT_DYNSEQ). Costs "
"about 10%% of embedding throughput; for A/B controls.")
args = parser.parse_args()
# Select profile set based on model role before any build function is called.
# build_with_trtexec and build_with_python_api both read the global PROFILES.
global PROFILES, DYNSEQ_MIN_SEQ
if args.role == "ner":
PROFILES = NER_PROFILES
elif args.role == "reranker":
PROFILES = RERANKER_PROFILES
# --variant takes precedence over the role-default PROFILES. Bucketed variants
# apply to embedding AND reranker (relation-CE length bucketing, 2026-07-10);
# rejected for `--role ner` (NER is single-shape). The shape is config-driven:
# --variant-batch/--variant-seq (forwarded by models/compile.cmd from the model's
# semantic_config.bucket_variants) win; else fall back to the embed-only table.
if args.variant is not None:
if args.role not in ("embedding", "reranker"):
print(
f"ERROR: --variant {args.variant!r} is incompatible with --role {args.role!r}; "
f"bucketed variants are for embedding/reranker only.",
file=sys.stderr,
)
sys.exit(1)
if args.variant_batch is not None and args.variant_seq is not None:
_vb, _vs = args.variant_batch, args.variant_seq
elif args.variant in BUCKET_VARIANT_PROFILES:
_vb, _vs = BUCKET_VARIANT_PROFILES[args.variant]
else:
print(
f"ERROR: --variant {args.variant!r} has no shape: pass --variant-batch/--variant-seq "
f"or use a known embed variant {list(BUCKET_VARIANT_PROFILES.keys())}.",
file=sys.stderr,
)
sys.exit(1)
PROFILES = [(_vb, _vs, _vb, _vs)]
print(f"[build_trt] --variant {args.variant} β rectangular ({_vb}, {_vs})", flush=True)
# Dynamic sequence for the embed grid. Keyed on the SHAPE, not the bucket
# name: relation-CE reuses s/m/l/xl at seq 64/128/256/320 and its opt
# points are not these. A shape that does not match the embed table
# stays rectangular.
_dyn = BUCKET_VARIANT_DYNSEQ.get(args.variant)
if _dyn and not args.no_dynseq and \
BUCKET_VARIANT_PROFILES.get(args.variant) == (_vb, _vs):
DYNSEQ_MIN_SEQ, _opt = _dyn
PROFILES = [(_vb, _vs, _vb, _opt)]
print(f"[build_trt] --variant {args.variant} β dynamic seq "
f"{DYNSEQ_MIN_SEQ}β{_opt}β{_vs} at batch {_vb}", flush=True)
# Dynamic-seq probe β applied last so it wins over both the rectangular
# --variant shape and the BUCKET_VARIANT_DYNSEQ table. This is the knob for
# sweeping a candidate grid; production builds get their shape from the
# table and leave it unset.
_dynseq = os.environ.get("DYNSEQ_OVERRIDE", "").strip()
if _dynseq:
try:
_db, _dmin, _dopt, _dmax = (int(x) for x in _dynseq.split(","))
PROFILES = [(_db, _dmax, _db, _dopt)]
DYNSEQ_MIN_SEQ = _dmin
print(
f"[build_trt] DYNSEQ probe: batch {_db} rectangular, seq {_dmin}β{_dopt}β{_dmax}",
flush=True,
)
except (ValueError, AttributeError) as _e:
print(f"[build_trt] WARN: bad DYNSEQ_OVERRIDE='{_dynseq}': {_e}", flush=True)
use_bf16 = args.bf16
use_fp8 = args.fp8 and not use_bf16
use_int8 = args.int8 and not args.no_int8 and not use_fp8 and not use_bf16
use_fp16 = args.fp16 and not use_bf16
use_per_layer_norm = args.per_layer_norm and use_int8 and not use_bf16
use_strongly_typed = False
cuda_arch = args.cuda_arch or get_cuda_arch()
if not cuda_arch:
print("ERROR: Cannot detect CUDA arch. Pass --cuda-arch sm_XXX", file=sys.stderr)
sys.exit(1)
# `--cuda-arch` names the FILE; TensorRT always builds for the GPU that is
# present. So the detected architecture is the ONLY one this machine may
# produce, and a request for any other is refused rather than honoured with
# a rename. There is no override: an engine carrying an architecture it was
# not built for fails `deserializeEngine` on the real card with no fallback,
# and it fails there, on a user's machine, not here. Every other arch is
# built on a host that has it (Selectel VMs for sm_75/86/89).
#
# This is not hypothetical: dist/ held 30 engines across four models named
# sm_75/sm_86/sm_89 that were byte-identical to their sm_120 twins.
_detected = get_cuda_arch()
if _detected and args.cuda_arch and args.cuda_arch != _detected:
print(f"ERROR: --cuda-arch {args.cuda_arch} but this machine is "
f"{_detected}. TensorRT cannot build for an absent architecture; "
f"the engine would be a mislabelled {_detected} plan. Build it on "
f"a {args.cuda_arch} host.", file=sys.stderr)
sys.exit(2)
if args.onnx:
onnx_path = args.onnx
if not os.path.exists(onnx_path):
print(f"ERROR: --onnx path not found: {onnx_path}", file=sys.stderr)
sys.exit(1)
else:
# Mode-aware ONNX discovery: FP8 build β prefer model_fp8qdt.onnx;
# INT8 β model_int8qdt.onnx; BF16 β plain model.onnx.
# Without this, --fp8 mode would pick up model_int8qdt.onnx whose
# Q/DQ nodes are INT8-typed, defeating the FP8 builder flag and
# silently producing an INT8 engine.
_discover_mode = "fp8" if use_fp8 else ("bf16" if use_bf16 else "int8")
onnx_path = find_onnx(args.model_dir, mode=_discover_mode)
if onnx_path is not None and use_fp8 and not onnx_is_fp8(onnx_path):
# FP8 requested but no model_fp8qdt.onnx β find_onnx fell back
# to INT8 Q/DQ ONNX. Per line 1124 logic this auto-enables
# both --int8 and --fp8 builder flags, but TRT will only run
# INT8 (no FP8 Q/DQ nodes in the file). The resulting engine
# is INT8 disguised as FP8. Warn loudly so the user can run
# `quantize_onnx.py --fp8` first.
print(f"WARNING: --fp8 requested but {os.path.basename(onnx_path)} is INT8 Q/DQ.")
print(f"WARNING: Resulting engine will be INT8 (no FP8 Tensor Cores).")
print(f"WARNING: Run `scripts/quantize_onnx.py --fp8 --model-dir {args.model_dir}` first.")
if not onnx_path:
print(f"ERROR: ONNX model not found in {args.model_dir}", file=sys.stderr)
sys.exit(1)
# β MAKE THE FP16 LANE ACTUALLY FP16.
#
# TRT 11 is STRONGLY-TYPED ONLY: the per-precision builder flags were removed and
# precision is read entirely from the ONNX tensor types. `catalog.py` routes
# rerankers and `trt_fp16_only` models here with `--no-int8` and labels the build
# FP16, but nothing converted the graph β so every engine on this lane shipped as
# FP32. Measured 2026-08-03 across the deployed set: relation-v1 Float=335 Half=0,
# reranker-v1 Float=325 Half=0, denoise-v1 Float=331 Half=0 (its catalog entry even
# says BF16). Only embed-v1 was genuinely quantized, because its ONNX carries Q/DQ
# nodes and strong typing therefore has something to read.
#
# Converting and rebuilding: relation 20.74 -> 5.44 ms/chunk (3.81x), reranker
# 21.14 -> 5.40 (3.91x), denoise 8.56 -> 3.60 (2.38x), each verified against the
# previously deployed engine at cos >= 0.999995 with argmax fully preserved.
# Engines halve on disk too. It is ~4x rather than the ~1.4x a DRAM argument
# predicts because FP32 matmuls do not use the tensor cores at all.
#
# The tag drives it, as it always did β this only makes the flag mean its label.
# Re-verify a converted model with:
# models/_compile/tools/build_fp16_engine.py --stage verify --model <name>
if not use_int8 and not use_fp8 and not use_bf16:
try:
sys.path.insert(0, os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(
os.path.abspath(__file__)))), "tools"))
from onnx_fp16 import ensure_fp16_onnx
onnx_path = ensure_fp16_onnx(onnx_path)
except Exception as e: # noqa: BLE001 - never block a build on the conversion
print(f"WARNING: FP16 ONNX conversion failed ({e}); "
f"building FP32 from {os.path.basename(onnx_path)}")
model_id = args.model_id or os.path.basename(os.path.normpath(args.model_dir))
if args.os_arch:
os_arch = args.os_arch
else:
import platform as _platform
_machine = _platform.machine().lower()
_is_arm = _machine in ("arm64", "aarch64")
if sys.platform == "win32":
os_arch = "win_arm64" if _is_arm else "win_x64"
elif sys.platform == "darwin":
os_arch = "mac_arm64"
else:
os_arch = "linux_arm64" if _is_arm else "linux_x64"
# Length-aware variant suffix slotted into filename BEFORE the runtime
# tag β keeps the established `..._{os_arch}_trt{VER}_{cuda_arch}.engine`
# cluster intact so existing CDN tooling continues to glob by suffix.
# Layout: `{model_id}-{variant}_{os_arch}_trt{VER}_{cuda_arch}.engine`.
variant_tag = f"-{args.variant}" if args.variant else ""
rt_tag = _trt_runtime_tag()
_refuse_trt_version_skew(rt_tag)
engine_path = os.path.join(args.model_dir, f"{model_id}{variant_tag}_{os_arch}_{rt_tag}_{cuda_arch}.engine")
if os.path.exists(engine_path) and not args.force:
size_mb = os.path.getsize(engine_path) / (1024 * 1024)
print(f"Engine already exists: {engine_path} ({size_mb:.1f} MB)")
print("Use --force to rebuild")
sys.exit(0)
if args.reset_timing_cache:
timing_cache_path = os.path.join(args.model_dir, "timing.cache")
if os.path.exists(timing_cache_path):
os.remove(timing_cache_path)
print(f" Timing cache deleted: {timing_cache_path} (fresh kernel selection)")
# Explicit Q/DQ ONNX detected: override precision mode based on Q/DQ flavour.
# INT8 Q/DQ path β STRONGLY_TYPED: prevents TRT's backward type-inference from propagating
# INT8 from DequantizeLinear's input to the shared Q/DQ scale initializer.
# ORT's quantize_static creates one scale initializer shared by both Q and DQ in each pair.
# In weakly-typed mode this causes "[SCALE] has invalid precision Int8, ignored" and disables
# W8A16 fusion (verified 363 warnings = 363 DQ nodes, TRT 10.16, sm_120).
# With STRONGLY_TYPED, TRT reads precision from ONNX data_type; no builder flags needed.
#
# FP8 Q/DQ path β also STRONGLY_TYPED (corrected 2026-05-19): the prior
# "FP8 STRONGLY_TYPED = 171 emb/s" datum was measured on a plain `model.onnx`
# WITHOUT Q/DQ nodes β TRT fell through to the generic codepath. With a
# real FP8 Q/DQ ONNX from ModelOpt, STRONGLY_TYPED reads FP8 precision +
# BF16 scale initializers directly from the graph, avoiding the
# weakly-typed validate-error trap: `bf16 precision has been set ... but
# bf16 is not configured in the builder` (TRT 10.16 silently rejects
# `config.set_flag(BF16)` when also FP8+INT8+FP16 are set, sm_120).
# No precision builder flags are set in this mode β TRT honours the
# ONNX types as-is.
#
# INT8 Q/DQ ONNX + --fp8 (legacy mixed path) β keep weakly-typed
# FP16+INT8+FP8: meaningful only when caller mixes an INT8 Q/DQ ONNX with
# --fp8 hint. Currently no production model hits this branch; preserved
# for backward compatibility.
if onnx_has_qdq(onnx_path):
use_bf16 = False
use_per_layer_norm = False
if onnx_is_fp8(onnx_path):
# FP8 Q/DQ ONNX β STRONGLY_TYPED reads everything from graph.
use_fp16 = False
use_int8 = False
use_fp8 = False # builder flags suppressed; FP8 comes from Q/DQ nodes
use_strongly_typed = True
print(f"[PTQ] Explicit FP8 Q/DQ ONNX β STRONGLY_TYPED (FP8 Tensor Cores + BF16 scales from ONNX)")
elif use_fp8:
# INT8 Q/DQ ONNX + --fp8: weakly-typed mixed path (legacy).
use_fp16 = True
use_int8 = True
use_strongly_typed = False
print(f"[PTQ] INT8 Q/DQ ONNX + --fp8 β FP16+INT8+FP8 (kFP8 Tensor Cores for Q/DQ layers)")
else:
# INT8 Q/DQ ONNX, current production path.
use_fp16 = False
use_int8 = False
use_strongly_typed = True
print(f"[PTQ] Explicit Q/DQ ONNX β STRONGLY_TYPED (W8A16 path, no [SCALE] type-inference warnings)")
if use_strongly_typed:
# Differentiate between INT8 Q/DQ (W8A16) and FP8 Q/DQ (E4M3+BF16).
# `onnx_is_fp8` is filename-based and was already evaluated before
# the flag-resolution block above.
precision = "STRONGLY_TYPED+FP8" if onnx_is_fp8(onnx_path) else "STRONGLY_TYPED+W8A16"
elif use_bf16:
precision = "BF16"
elif use_fp8:
precision = "FP8+FP16"
elif use_int8 and use_per_layer_norm:
precision = "INT8+BF16-norm"
elif use_int8:
precision = "FP16+INT8"
else:
precision = "FP16"
print(f"Building TRT engine: {cuda_arch} ({precision})")
print(f" ONNX: {onnx_path}")
print(f" Engine: {engine_path}")
print()
ok = False
# Per-layer-norm mode requires Python API (trtexec has no per-layer precision CLI).
# FP8 (weakly-typed kFP8+kFP16) and all other modes can use trtexec.
trtexec = None
if not use_per_layer_norm:
trtexec = shutil.which("trtexec")
if not trtexec:
for p in [
"/usr/bin/trtexec",
"/usr/local/bin/trtexec",
]:
if os.path.isfile(p) and os.access(p, os.X_OK):
trtexec = p
break
if not trtexec:
import glob
for pattern in [
"/usr/local/TensorRT-*/bin/trtexec",
"/opt/tensorrt/bin/trtexec",
"/usr/local/cuda/bin/trtexec",
]:
matches = glob.glob(pattern)
if matches:
trtexec = matches[-1] # latest version
break
if trtexec:
print(f"Using trtexec: {trtexec}")
ok = build_with_trtexec(onnx_path, engine_path, trtexec_bin=trtexec)
else:
print("trtexec not in PATH β using Python TensorRT API")
ok = build_with_python_api(onnx_path, engine_path)
# TRT 11 has a SINGLE build path: strongly-typed, precision read from the
# ONNX Q/DQ nodes. The TRT-10 fallback cascade (retry as weakly-typed
# FP16+INT8 / BF16 / FP8 / calibrated-INT8) was removed along with those
# builder flags + the calibrator β there is nothing weaker to fall back to.
# If the strongly-typed build fails, it fails; the caller surfaces the TRT
# error from the log above.
if not ok or not os.path.exists(engine_path):
print("ERROR: Engine build failed", file=sys.stderr)
sys.exit(1)
size_mb = os.path.getsize(engine_path) / (1024 * 1024)
print(f"\nSUCCESS: {engine_path} ({size_mb:.1f} MB)")
if __name__ == "__main__":
main()
#!/usr/bin/env python3
"""Build TensorRT engine from ONNX model with multi-profile bucketing.
Usage:
python scripts/build_trt_engine.py [--model-dir PATH] [--cuda-arch sm_120]
Two modes:
1. trtexec (if in PATH) β NVIDIA's official CLI tool
2. Python TensorRT API (fallback) β uses tensorrt pip package
Profiles (3 buckets matching tensorrt_runtime.zig default_profiles):
0: batch 1-32, seq 1-32 (opt: 16x32) β short entity names
1: batch 1-64, seq 1-64 (opt: 32x64) β entity descriptions
2: batch 1-128, seq 1-128 (opt: 128x128) β docstrings / function bodies
Output: {model_id}_{os_arch}_trt{VER}_{cuda_arch}.engine in the model directory
(os_arch = win_x64 / linux_x64 / linux_arm64 / mac_arm64;
VER = TensorRT major.minor from vendor/tensorrt/VERSION, e.g. trt11.0).
A serialized plan is keyed on {GPU arch x OS x TRT version} and TensorRT offers
no compat fallback on deserialize, so the version belongs in the filename next
to the arch: the daemon then resolves an engine it can actually load instead of
opening a same-named plan from another TRT and failing inside the runtime.
"""
import argparse
import os
import shutil
import subprocess
import sys
import warnings
from pathlib import Path
# Force UTF-8 output on Windows (default console encoding is CP1251/CP1252)
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
if hasattr(sys.stderr, "reconfigure"):
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
# Auto-install pip deps on first run when this script is invoked directly
# (i.e. without going through models/code-daemon-embed/run.ps1 / run.sh which
# already install everything). Idempotent β skips packages that are already importable.
if os.environ.get("ULTRACODE_NO_BOOTSTRAP") != "1":
# Repo root by walking up to build.zig, not by counting `..` β this file has
# moved once already (scripts/ -> models/_compile/backends/workers/) and a
# hardcoded depth breaks silently, since a missing directory just means "no
# bootstrap available".
_root = Path(__file__).resolve()
for _ in range(6):
_root = _root.parent
if (_root / "build.zig").is_file():
break
_bootstrap_dir = _root / "models" / "code-daemon-embed"
if (_bootstrap_dir / "_bootstrap.py").exists():
sys.path.insert(0, str(_bootstrap_dir))
try:
from _bootstrap import ensure_deps, ensure_nvidia_deps
# onnx is needed for ensure_single_file_onnx() and proto inspection;
# torch is not required by this script β skip the torch install.
ensure_deps(extra=["onnx"], torch_required=False)
# tensorrt + cuda-python (NVIDIA-only). No-op on macOS / CPU hosts.
ensure_nvidia_deps()
except Exception as _bs_err:
print(f"[bootstrap] WARNING: {_bs_err}", flush=True)
finally:
sys.path.remove(str(_bootstrap_dir))
# Suppress TRT deprecation warnings: implicit INT8 calibration (superseded by QAT/explicit
# quantization in TRT 10.1+) and per-layer precision (superseded by strong typing in TRT 10.12+).
# Replacements require ONNX QAT nodes / STRONGLY_TYPED mode β not applicable here.
warnings.filterwarnings("ignore", message=".*Deprecated.*", category=DeprecationWarning)
warnings.filterwarnings("ignore", message=".*deprecated.*", category=DeprecationWarning)
def ensure_single_file_onnx(onnx_path):
"""Convert split ONNX (model.onnx + model.onnx_data) to single-file if needed."""
data_file = onnx_path + "_data"
if not os.path.exists(data_file):
return onnx_path
merged_path = onnx_path.replace(".onnx", "_merged.onnx")
if os.path.exists(merged_path):
merged_size = os.path.getsize(merged_path)
if merged_size < 1024 or merged_size > 2_000_000_000:
# Empty/corrupted or >2GB (protobuf can't parse) β delete stale cache
os.remove(merged_path)
print(f" Removed stale merged ONNX ({merged_size / 1e6:.0f} MB)")
else:
merged_mtime = os.path.getmtime(merged_path)
if (merged_mtime > os.path.getmtime(onnx_path) and
merged_mtime > os.path.getmtime(data_file)):
print(f" Split ONNX: using cached merged file")
return merged_path
try:
import onnx
print(f" Split ONNX detected: merging into single file...")
model = onnx.load(onnx_path, load_external_data=True)
onnx.save(model, merged_path)
merged_size = os.path.getsize(merged_path)