File size: 60,475 Bytes
40da26e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 | #!/usr/bin/env python3
"""Build TensorRT engine from ONNX model with multi-profile bucketing.
Usage:
python scripts/build_trt_engine.py [--model-dir PATH] [--cuda-arch sm_120]
Two modes:
1. trtexec (if in PATH) β NVIDIA's official CLI tool
2. Python TensorRT API (fallback) β uses tensorrt pip package
Profiles (3 buckets matching tensorrt_runtime.zig default_profiles):
0: batch 1-32, seq 1-32 (opt: 16x32) β short entity names
1: batch 1-64, seq 1-64 (opt: 32x64) β entity descriptions
2: batch 1-128, seq 1-128 (opt: 128x128) β docstrings / function bodies
Output: {model_id}_{os_arch}_trt{VER}_{cuda_arch}.engine in the model directory
(os_arch = win_x64 / linux_x64 / linux_arm64 / mac_arm64;
VER = TensorRT major.minor from vendor/tensorrt/VERSION, e.g. trt11.0).
A serialized plan is keyed on {GPU arch x OS x TRT version} and TensorRT offers
no compat fallback on deserialize, so the version belongs in the filename next
to the arch: the daemon then resolves an engine it can actually load instead of
opening a same-named plan from another TRT and failing inside the runtime.
"""
import argparse
import os
import shutil
import subprocess
import sys
import warnings
from pathlib import Path
# Force UTF-8 output on Windows (default console encoding is CP1251/CP1252)
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
if hasattr(sys.stderr, "reconfigure"):
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
# Auto-install pip deps on first run when this script is invoked directly
# (i.e. without going through models/code-daemon-embed/run.ps1 / run.sh which
# already install everything). Idempotent β skips packages that are already importable.
if os.environ.get("ULTRACODE_NO_BOOTSTRAP") != "1":
# Repo root by walking up to build.zig, not by counting `..` β this file has
# moved once already (scripts/ -> models/_compile/backends/workers/) and a
# hardcoded depth breaks silently, since a missing directory just means "no
# bootstrap available".
_root = Path(__file__).resolve()
for _ in range(6):
_root = _root.parent
if (_root / "build.zig").is_file():
break
_bootstrap_dir = _root / "models" / "code-daemon-embed"
if (_bootstrap_dir / "_bootstrap.py").exists():
sys.path.insert(0, str(_bootstrap_dir))
try:
from _bootstrap import ensure_deps, ensure_nvidia_deps
# onnx is needed for ensure_single_file_onnx() and proto inspection;
# torch is not required by this script β skip the torch install.
ensure_deps(extra=["onnx"], torch_required=False)
# tensorrt + cuda-python (NVIDIA-only). No-op on macOS / CPU hosts.
ensure_nvidia_deps()
except Exception as _bs_err:
print(f"[bootstrap] WARNING: {_bs_err}", flush=True)
finally:
sys.path.remove(str(_bootstrap_dir))
# Suppress TRT deprecation warnings: implicit INT8 calibration (superseded by QAT/explicit
# quantization in TRT 10.1+) and per-layer precision (superseded by strong typing in TRT 10.12+).
# Replacements require ONNX QAT nodes / STRONGLY_TYPED mode β not applicable here.
warnings.filterwarnings("ignore", message=".*Deprecated.*", category=DeprecationWarning)
warnings.filterwarnings("ignore", message=".*deprecated.*", category=DeprecationWarning)
def ensure_single_file_onnx(onnx_path):
"""Convert split ONNX (model.onnx + model.onnx_data) to single-file if needed."""
data_file = onnx_path + "_data"
if not os.path.exists(data_file):
return onnx_path
merged_path = onnx_path.replace(".onnx", "_merged.onnx")
if os.path.exists(merged_path):
merged_size = os.path.getsize(merged_path)
if merged_size < 1024 or merged_size > 2_000_000_000:
# Empty/corrupted or >2GB (protobuf can't parse) β delete stale cache
os.remove(merged_path)
print(f" Removed stale merged ONNX ({merged_size / 1e6:.0f} MB)")
else:
merged_mtime = os.path.getmtime(merged_path)
if (merged_mtime > os.path.getmtime(onnx_path) and
merged_mtime > os.path.getmtime(data_file)):
print(f" Split ONNX: using cached merged file")
return merged_path
try:
import onnx
print(f" Split ONNX detected: merging into single file...")
model = onnx.load(onnx_path, load_external_data=True)
onnx.save(model, merged_path)
merged_size = os.path.getsize(merged_path)
if merged_size > 2_000_000_000:
# >2GB: protobuf can't parse it back, trtexec may fail. Use original + external data.
os.remove(merged_path)
print(f" Merged {merged_size / 1e6:.0f} MB > 2GB limit, using split ONNX (parse_from_file handles external data)")
return onnx_path
print(f" Merged: {merged_size / (1024*1024):.1f} MB")
return merged_path
except Exception as e:
print(f" Split ONNX merge failed: {e}")
return onnx_path
# Profile specs: (max_batch, max_seq, opt_batch, opt_seq)
#
# Single rectangular profile per model type β no multi-profile bucketing.
# The daemon accumulates up to batch=128 before dispatch; buildEmbedInput
# caps at 420 chars β 64 tokens so seq=64 covers all real traffic.
# Fewer profiles = less VRAM, faster compile, trivial selectProfile (always 0).
# Multi-profile builds were tried and produce slower kernels for the seq=64 path
# on Gemma sm_120 (autotuner picks kernels optimized for the broader shape range).
#
# Profile specs: (max_batch, max_seq, opt_batch, opt_seq).
#
# 2026-05-05 β half-batch-zero saga resolved. The "TRT sm_120 regression"
# we chased turned out to be a Zig-side dtype mismatch: engine input
# binding is INT64 (verified via getTensorDtype), but our worker copied
# host buffers as INT32 stride. Engine read 8 bytes per token from a
# buffer where we'd written 4 β got 2Γ rows of garbage from uninit
# memory, manifesting as "second half of every batch is zero". Fixed
# in tensorrt_runtime.zig + embed_worker.zig (i32 β i64 throughout the
# TRT input chain). Engine itself is fine; restore full 128 batch.
PROFILES = [
(128, 64, 128, 64), # 0: rectangular fast path β batch 128, seq 64
]
RECTANGULAR_PROFILE = True
# Dynamic-seq probe (2026-07-28). Set `DYNSEQ_OVERRIDE="batch,min_seq,opt_seq,max_seq"`
# to build ONE engine whose SEQUENCE dimension is a range while the batch stays
# rectangular. Purpose: a chunk could then be dispatched at its own longest text
# instead of the bucket's ceiling β live telemetry says 15.3% of GPU token-work
# is padding (avg 42/48, 56/64, 70/80, 95/128). The question this answers is
# whether TRT's broader-range kernel choice costs more than the padding it saves;
# the file header already records that multi-PROFILE builds regressed seq=64 on
# sm_120. Applied AFTER --variant in main() so the engine still gets a suffixed
# filename. Production builds leave it unset.
DYNSEQ_MIN_SEQ = None
# βββ Experimental switches (hardcoded) ββββββββββββββββββββββββββββββββββ
# Project preference: in-source const flip is cleaner than env-var dance
# for one-off experiments. Each switch documents its evidence + revert.
# Flip the value, save, run models/compile or build_trt_engine directly.
# Builder workspace pool. Default 4 GB; bump to 8-12 GB to widen TRT
# tactic search on autotuning. Build-time only β does NOT affect engine
# size or runtime VRAM. Risk: build OOM if value > free VRAM at build time.
# REVERT: change back to 4.
TRT_WORKSPACE_GB = 8
# TRT max_aux_streams config.
# 0 β workaround for the 2026-04-28 "5 valid + N zeros" sm_120 codegen
# bug (TRT 10.13.3). Forces single-stream kernel generation.
# 4 β TRT default. Researcher confirmed 2026-05-19 that post-fusion
# embedding graphs (BGE/Qwen3/Gemma) are too linear for aux streams
# to help, and TRT 10.16 gracefully self-disables ("Multi stream is
# disabled because the stream assignment failed" warning). Either
# value produces functionally identical engines on this family.
# Kept as probe knob β REVERT: 0.
TRT_MAX_AUX_STREAMS = 0
# Env override for sweep experiments. Format: RECT_PROFILE_OVERRIDE="batch,seq"
# β collapses PROFILES to a single (batch, seq, batch, seq) entry. Production
# builds leave this unset. Example: RECT_PROFILE_OVERRIDE="32,32".
_rect_override = os.environ.get("RECT_PROFILE_OVERRIDE", "").strip()
if _rect_override:
try:
_b, _s = (int(x) for x in _rect_override.split(","))
PROFILES = [(_b, _s, _b, _s)]
print(f"[build_trt] PROFILES overridden via env: rectangular ({_b}, {_s})", flush=True)
except (ValueError, AttributeError) as _e:
print(f"[build_trt] WARN: bad RECT_PROFILE_OVERRIDE='{_rect_override}': {_e}", flush=True)
# Length-aware bucket variants β when `--variant {s,m,l}` is passed, PROFILES
# collapses to that variant's rectangular (max_batch, max_seq). Single source
# of truth lives in `models/code-daemon-embed/embed_bucket_config.py`. We
# duplicate the tuples here to avoid forcing the build to import from a model
# subdirectory at parse-time (the bootstrap path may not yet be on sys.path).
# `models/_compile/preflight_bucket_sync.py` runs as a pre-build gate to catch drift.
BUCKET_VARIANT_PROFILES = {
# 2026-07-27: grid 40/64/128 β 32/48/80/128, measured on real engines
# (see the header comment in src/inference/embed_bucket_config.zig).
"s": (96, 48),
"m": (128, 64),
"l": (128, 80),
"xl": (256, 128),
}
# (min_seq, opt_seq) per embed bucket β the SEQUENCE dimension is a range while
# the batch stays rectangular, so a chunk is dispatched at its own longest text
# instead of the bucket ceiling. Live telemetry put 15.3% of GPU token-work in
# sequence padding; recovering it measured +10.2% end to end (9,738 -> 10,731
# emb/s on mysql-server) and is quality-neutral by construction β padding is
# masked in attention, verified at cosine 1.000000 on 128 rows of real ids.
#
# Two rules produced these numbers and neither is guessable:
# - `opt` sits at the bucket's histogram-weighted MEAN, not its ceiling. The
# tax is paid for being far from the opt point, not for being dynamic: one
# wide 16β64β128 engine costs +0.8% at seq 64 and +15.4% at seq 80.
# - `min` is the bucket FLOOR, not the observed corpus minimum β `s` gets 8
# rather than the measured 24, because the router must accept a 3-token text.
#
# This lived in a DYNSEQ_OVERRIDE env var and was deployed by copying files over
# the rectangular ones. A later `models/compile` run rebuilt them rectangular and
# the +10.2% was silently gone; the engines still loaded, still returned 768-dim
# vectors, and reported 54 layers instead of 69 where nobody was looking. Hence
# the table: a plain build now produces the shape that is meant to ship.
BUCKET_VARIANT_DYNSEQ = {
"s": (8, 48),
"m": (56, 56),
"l": (72, 72),
"xl": (88, 96),
}
# NER single profile: batch=32 (NER worker --batch-size 32), seq=128.
# Bumped from seq=64 (2026-05-05) β ~20% of incoming chunks exceed 64
# tokens (long entity descriptions, function bodies, docstrings), causing
# `SetShapeFailed` at runtime and DocGraph falling back to rules-only for
# those entities. The OV NER variants ship as `_b32_s128`, so the ONNX
# already supports seq=128. Opt point sits at 96 β middle of the 1..128
# range, biasing the autotuner toward the typical mass of inputs without
# regressing the long tail.
NER_PROFILES = [
# 0: NER main path β RECTANGULAR (max == opt) to avoid an invalid
# min<=opt<=max profile. The previous value (32, 128, 32, 96) set
# opt_seq=96 < max_seq=128, which under RECTANGULAR_PROFILE (min==max==128)
# makes opt_seq < min_seq β TRT 11 rejects with "Invalid optimization
# profiles ... ensure MIN<=OPT<=MAX". The NER worker always pads to
# batch=32 seq=128, so a static 32x128 profile is correct.
(32, 128, 32, 128),
]
# Reranker single profile: batch=16 (reranker worker max_batch=16, matches the OV engine
# `_b16_s256` and models/compile.cmd's $ovChkBatch=16), seq=256. The cross-encoder scores
# (query, document) PAIRS, which are long, so seq must be 256 β NOT the embedding default
# (128, 64). Without a reranker-specific profile, `--role reranker` fell through to the
# embedding PROFILES (128, 64); the engine bound seq=64 while handleRerank pads to
# max_seq_len=256 β SetShapeFailed at runtime β search degraded to no-rerank. RECTANGULAR
# (min==opt==max) per handleRerank's rectangular-engine contract (embed_worker.zig:4867).
RERANKER_PROFILES = [
(16, 256, 16, 256),
]
def _installed_trt_tag():
"""`trt{major}.{minor}` of the TensorRT in THIS interpreter, or None."""
try:
import tensorrt as trt
parts = str(trt.__version__).split(".")
except Exception:
return None
return f"trt{parts[0]}.{parts[1]}" if len(parts) >= 2 and parts[0] else None
def _pinned_trt_tag():
"""`trt{major}.{minor}` from `vendor/tensorrt/VERSION`, or None if no repo.
This is the pin build.zig turns into the daemon's `runtime_versions.trt_tag`,
so naming from it guarantees the engine we write is the engine the worker
looks for. Absent when the script runs standalone β the cloud sm_86/sm_89
builder (models/_cloud_shared/build-tools/build_engines.sh) copies only this
.py onto a bare VM, with no vendor/ tree to read.
"""
d = os.path.dirname(os.path.abspath(__file__))
for _ in range(6): # walk up to repo root
pin = os.path.join(d, "vendor", "tensorrt", "VERSION")
if os.path.isfile(pin):
try:
parts = open(pin).read().strip().split(".")
except OSError:
return None
return f"trt{parts[0]}.{parts[1]}" if len(parts) >= 2 and parts[0] else None
parent = os.path.dirname(d)
if parent == d:
return None
d = parent
return None
def _trt_runtime_tag():
"""Runtime-version tag embedded in every engine filename, e.g. `trt11.0`.
Prefers the repo pin (what the DAEMON will deserialize with); falls back to
the TensorRT actually doing the build, so a standalone/cloud run still
produces a correctly-labelled engine instead of failing. Only when neither is
knowable do we stop: an untagged engine is invisible to the daemon, and TRT
builds are far too slow to discover that afterwards.
"""
tag = _pinned_trt_tag() or _installed_trt_tag()
if tag:
return tag
print("ERROR: cannot determine the TensorRT version β neither vendor/tensorrt/VERSION")
print(" nor an importable `tensorrt` package. Engine filenames carry the")
print(" TensorRT major.minor and the daemon only loads its own version.")
sys.exit(1)
def _refuse_trt_version_skew(rt_tag):
"""Stop when the TensorRT building this engine is not the pinned one.
Matters more here than anywhere else: the plan is produced by the `tensorrt`
package in THIS interpreter but deserialized by the vendored nvinfer the
daemon links. If they differ, the file gets a name promising a runtime that
cannot read it β a load failure the version tag exists to prevent.
This was a WARNING until 2026-09-12. It printed on every Linux build for
weeks while every Linux engine was built by 11.2 and named trt11.0
(`models/_compile/deps/wsl.py:_trt_pin`), because nothing consumed it. There
is no legitimate skew: a patch-level difference yields the same tag and never
reaches this branch, and a standalone cloud run has no pin, so `rt_tag` IS the
installed tag there. A missing package is handled elsewhere.
"""
installed_tag = _installed_trt_tag()
if installed_tag and installed_tag != rt_tag:
print(f"ERROR: TensorRT version skew β this interpreter has {installed_tag} "
f"but the engine would be named {rt_tag} (from vendor/tensorrt/VERSION).")
print(f" The daemon deserializes with the pinned runtime and would "
f"REJECT the plan. Install tensorrt-cu13 at the pin, or bump the pin.")
sys.exit(1)
def get_cuda_arch():
"""Detect CUDA compute capability via nvidia-smi."""
try:
out = subprocess.check_output(
["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"],
text=True,
).strip().split("\n")[0]
major, minor = out.split(".")
return f"sm_{major}{minor}"
except Exception:
return None
def _onnx_is_valid(path):
"""Quick-check that an ONNX file is parseable (not truncated).
Files >1 MB that fail onnx.load are assumed valid: Python protobuf has a 2 GB
parse limit but trtexec uses the C++ ONNX parser which has no such limit."""
try:
import onnx
onnx.load(path, load_external_data=False)
return True
except Exception as e:
try:
size = os.path.getsize(path)
if size > 1024 * 1024:
print(f" NOTE: onnx.load failed ({e}), file is {size / (1024*1024):.0f} MB "
f"β assuming valid (trtexec uses C++ parser, no 2 GB limit)")
return True
except Exception:
pass
return False
def _has_all_external_data(onnx_path):
"""Return True if every external-data file referenced by the ONNX header exists
in the same directory. Returns True (optimistic) when the header itself cannot
be loaded (e.g. 2 GB+ merged file β no external refs to check)."""
try:
import onnx as _onnx
header = _onnx.load(onnx_path, load_external_data=False)
except Exception:
return True # can't check β assume ok (2 GB+ merged, no external refs)
onnx_dir = os.path.dirname(os.path.abspath(onnx_path))
for t in header.graph.initializer:
if t.data_location == _onnx.TensorProto.EXTERNAL:
for entry in t.external_data:
if entry.key == "location":
if not os.path.exists(os.path.join(onnx_dir, entry.value)):
return False
return True
def find_onnx(model_dir, mode="int8"):
"""Find ONNX model in model directory, with validation and cache fallback.
`mode` controls preference order:
- "fp8": prefer `model_fp8qdt.onnx` (ModelOpt FP8 PTQ output) over INT8.
- "int8" (default): prefer `model_int8qdt.onnx` (ModelOpt INT8 PTQ output).
- "bf16": prefer plain `model.onnx`.
Falls back through the alternate-precision PTQ ONNX before raw `model.onnx`
so that a misconfigured catalog still produces a build (TRT will pick up
whatever Q/DQ nodes are in the file).
"""
# Last-resort fallback: the compiler's download cache for third-party models
# (`models/_compile/.cache/models/<id>/`). The compiler itself always passes
# `--onnx` explicitly and the cloud builder stages the ONNX into --model-dir,
# so this is only reached by a hand-run invocation.
#
# It used to be derived as `<this script's dir>/.cache/<id>`, which pointed
# at `scripts/.cache` while the script lived there and became a path that can
# never exist once it moved.
model_name = os.path.basename(os.path.normpath(model_dir))
_pkg_dir = os.path.dirname(os.path.abspath(__file__)) # β¦/backends/workers
_compile_root = os.path.dirname(os.path.dirname(_pkg_dir)) # β¦/models/_compile
cache_dir = os.environ.get("ULTRACODE_COMPILE_CACHE") or \
os.path.join(_compile_root, ".cache")
cache_path = os.path.join(cache_dir, "models", model_name, "model.onnx")
int8_candidates = [
os.path.join(model_dir, "onnx", "model_int8qdt.onnx"),
os.path.join(model_dir, "model_int8qdt.onnx"),
os.path.join(cache_dir, "models", model_name, "model_int8qdt.onnx"),
]
fp8_candidates = [
os.path.join(model_dir, "onnx", "model_fp8qdt.onnx"),
os.path.join(model_dir, "model_fp8qdt.onnx"),
os.path.join(cache_dir, "models", model_name, "model_fp8qdt.onnx"),
]
plain_candidates = [
os.path.join(model_dir, "model.onnx"),
os.path.join(model_dir, "onnx", "model.onnx"),
cache_path,
]
if mode == "fp8":
candidates = fp8_candidates + int8_candidates + plain_candidates
elif mode == "bf16":
candidates = plain_candidates + int8_candidates + fp8_candidates
else:
candidates = int8_candidates + plain_candidates + fp8_candidates
for p in candidates:
if os.path.exists(p):
if not _onnx_is_valid(p):
print(f" WARNING: {p} is corrupted (truncated?), trying next candidate")
continue
if not _has_all_external_data(p):
print(f" NOTE: {p} has missing external data files β trying next candidate")
continue
return p
return None
def onnx_has_qdq(onnx_path):
"""True if the ONNX model contains explicit Q/DQ nodes (PTQ quantized).
When Q/DQ nodes are present, TRT must have a matching precision builder
flag enabled (INT8 for int8qdt, FP8 for fp8qdt). Without it, TRT silently
ignores DequantizeLinear/QuantizeLinear and builds a plain BF16/FP16
engine, wasting the calibration work.
Detection by filename is sufficient β quantize_onnx.py always writes
'model_int8qdt.onnx' (INT8 mode) or 'model_fp8qdt.onnx' (FP8 mode).
"""
name = os.path.basename(onnx_path)
return any(tag in name for tag in ("int8qdt", "fp8qdt", "nvfp4qdt", "int4qdt"))
def onnx_is_fp8(onnx_path):
"""True if the ONNX model is a ModelOpt FP8 PTQ output (fp8qdt naming).
Used to distinguish FP8-quantized ONNX (needs --fp8 builder flag) from
INT8-quantized ONNX (needs --int8). When neither, fallback paths apply.
"""
return "fp8qdt" in os.path.basename(onnx_path)
def build_with_trtexec(onnx_path, engine_path, trtexec_bin="trtexec"):
"""Build engine using trtexec CLI (TRT 11: strongly-typed only)."""
onnx_path = ensure_single_file_onnx(onnx_path)
cmd = [trtexec_bin, f"--onnx={onnx_path}", f"--saveEngine={engine_path}"]
# TRT 11: precision is read entirely from the ONNX Q/DQ nodes. Strongly typed
# is the only typing mode β the per-precision flags (--int8/--fp16/--bf16/--fp8)
# and the INT8 calibrator (--calib) were removed. `--stronglyTyped` is not
# passed: TRT 11.3 release notes list it as a flag with no effect.
# Timing cache: speeds up rebuilds dramatically (minutes β seconds)
timing_cache = os.path.join(os.path.dirname(engine_path), "timing.cache")
cmd.append(f"--timingCacheFile={timing_cache}")
# Max builder optimization level for best engine quality
cmd.append("--builderOptimizationLevel=5")
# 2026-04-28 β sm_120 aux-stream regression fix (researcher rec #4 in
# .autodoc/archive/inference-backends.md). TRT 10.13.3 / 10.16.1 release notes
# document a multi-stream codegen bug on consumer Blackwell (sm_120)
# that affects pooling/attention subgraphs and produces partial
# outputs (the embeddinggemma "5 valid + N-5 zeros" symptom matches
# this signature: kernel runs but writes zeros to trailing rows).
# Disabling aux streams bypasses the buggy codegen path; cost is
# one stream of dispatch concurrency we don't currently exploit.
#
# 2026-05-19 β value lives at module top as TRT_MAX_AUX_STREAMS const.
# Flip there + rebuild to probe (researcher confirmed it's effectively
# a no-op on embedding models; TRT 10.16 self-disables anyway).
cmd.append(f"--maxAuxStreams={TRT_MAX_AUX_STREAMS}")
if TRT_MAX_AUX_STREAMS != 0:
print(f" [build_trt] TRT_MAX_AUX_STREAMS={TRT_MAX_AUX_STREAMS} β sm_120 workaround DISABLED")
# `--useCudaGraph` / `--useSpinWait` were passed here until 2026-09-22. They
# only shaped trtexec's post-build benchmark, never the saved engine, and
# TRT 11.3 release notes list both as flags with no effect β so a benchmark
# line printed by an 11.3 trtexec is measured WITHOUT graphs and does not
# compare with 10.16/11.0 logs. The runtime counterpart was measured and
# left off: `cuda_graphs_enabled` in src/inference/tensorrt_runtime.zig.
for max_b, max_s, opt_b, opt_s in PROFILES:
cmd.extend([
f"--minShapes=input_ids:1x1,attention_mask:1x1",
f"--optShapes=input_ids:{opt_b}x{opt_s},attention_mask:{opt_b}x{opt_s}",
f"--maxShapes=input_ids:{max_b}x{max_s},attention_mask:{max_b}x{max_s}",
])
print(f" trtexec command: {' '.join(cmd[:5])} ...")
result = subprocess.run(cmd)
return result.returncode == 0
def _add_nvinfer_to_path():
"""Add vendor/tensorrt/lib AND pip tensorrt_libs to DLL search path."""
added = []
# 1. vendor/tensorrt/lib (our bundled runtime DLLs).
# This file lives at models/_compile/backends/workers/, so the repo root is
# four levels up. It used to be one (`scripts/../vendor`), which stopped
# resolving the moment the script moved β silently, because the directory
# simply "was not there" and the pip tensorrt_libs below usually covered it.
_here = os.path.dirname(os.path.abspath(__file__))
_repo_root = os.path.abspath(os.path.join(_here, "..", "..", "..", ".."))
vendor_lib = os.path.join(_repo_root, "vendor", "tensorrt", "lib")
if os.path.isdir(vendor_lib):
added.append(vendor_lib)
# 2. pip tensorrt_libs (contains nvinfer_plugin_10.dll needed for builder)
try:
import site
for sp in site.getsitepackages() + [site.getusersitepackages()]:
trt_libs = os.path.join(sp, "tensorrt_libs")
if os.path.isdir(trt_libs):
added.append(trt_libs)
break
except Exception:
pass
# Register all paths
for lib_dir in added:
if sys.platform == "win32" and hasattr(os, "add_dll_directory"):
try:
os.add_dll_directory(lib_dir)
except OSError:
pass
os.environ["PATH"] = lib_dir + os.pathsep + os.environ.get("PATH", "")
return added[0] if added else None
def _import_tensorrt():
"""Import tensorrt, handling the cu12_bindings layout."""
_add_nvinfer_to_path()
# Try direct import first (tensorrt meta-package installed)
try:
import tensorrt
return tensorrt
except (ImportError, ModuleNotFoundError, FileNotFoundError, OSError):
pass
# Try bindings-only import (tensorrt_cu12_bindings installed)
try:
from tensorrt_bindings import tensorrt
return tensorrt
except (ImportError, ModuleNotFoundError, FileNotFoundError, OSError):
pass
return None
def _has_cuda_runtime():
"""Check if CUDA runtime is available (libcudart loadable)."""
import ctypes
for lib in ["libcudart.so", "libcudart.so.13", "libcudart.so.12", "libcudart.so.11",
"cudart64_13.dll", "cudart64_12.dll", "cudart64_110.dll"]:
try:
ctypes.cdll.LoadLibrary(lib)
return True
except OSError:
pass
# Also check if nvcc is in PATH (CUDA toolkit installed)
if shutil.which("nvcc"):
return True
return False
def ensure_tensorrt_package():
"""Auto-install tensorrt Python bindings + libs if missing."""
trt = _import_tensorrt()
if trt is not None:
return True
if not _has_cuda_runtime():
print(" tensorrt not available: CUDA runtime not found (no libcudart/nvcc)", file=sys.stderr)
print(" Install CUDA toolkit or add trtexec to PATH", file=sys.stderr)
return False
print(" tensorrt not found β installing via pip...")
nvidia_idx = "https://pypi.nvidia.com"
# Detect CUDA major version to pick matching tensorrt build. Mixing cu12
# and cu13 packages produces version-mismatch errors at engine deserialize
# because the engine plan encodes the build-time native DLL version, and
# `import tensorrt` then resolves to a different package's DLL at runtime.
cuda_major = None
try:
nvsmi = subprocess.check_output(
["nvidia-smi"], stderr=subprocess.DEVNULL, text=True, timeout=5,
)
import re
m = re.search(r"CUDA Version:\s*(\d+)\.(\d+)", nvsmi)
if m:
cuda_major = int(m.group(1))
except Exception:
pass
if cuda_major and cuda_major >= 13:
candidates = [
["tensorrt-cu13-bindings==11.0.0.114", "tensorrt-cu13-libs==11.0.0.114"],
["tensorrt-cu13-bindings", "tensorrt-cu13-libs"],
]
else:
candidates = [
["tensorrt-cu12-bindings==11.0.0.114", "tensorrt-cu12-libs==11.0.0.114"],
["tensorrt-cu12-bindings", "tensorrt-cu12-libs"],
]
for pkgs in candidates:
result = subprocess.run(
[sys.executable, "-m", "pip", "install", "--quiet",
"--only-binary", ":all:",
"--extra-index-url", nvidia_idx] + pkgs,
capture_output=True, text=True,
)
if result.returncode == 0:
break
else:
# Last resort: try the meta-package
subprocess.run(
[sys.executable, "-m", "pip", "install", "--quiet",
"--only-binary", ":all:",
"--extra-index-url", nvidia_idx,
"tensorrt"],
capture_output=True, text=True,
)
trt = _import_tensorrt()
if trt is not None:
print(f" tensorrt {trt.__version__} ready")
return True
print(" tensorrt still not importable after install", file=sys.stderr)
return False
def _upgrade_onnx_opset(onnx_path, target_opset=17):
"""Upgrade ONNX model to target opset if needed (fixes LayerNorm FP16 overflow)."""
# Handle split ONNX first β skip upgrade if merge fails (>2GB models)
data_file = onnx_path + "_data"
if os.path.exists(data_file):
merged = ensure_single_file_onnx(onnx_path)
if merged == onnx_path:
# Merge failed (protobuf 2GB limit) β skip opset upgrade, TRT will parse directly
print(f" Opset upgrade skipped: split ONNX too large to merge")
return onnx_path
onnx_path = merged
else:
onnx_path = ensure_single_file_onnx(onnx_path)
try:
import onnx
from onnx import version_converter
model = onnx.load(onnx_path)
current_opset = model.opset_import[0].version if model.opset_import else 0
if current_opset >= target_opset:
return onnx_path # already sufficient
print(f" Upgrading ONNX opset {current_opset} β {target_opset} (fixes LayerNorm FP16 overflow)")
model = version_converter.convert_version(model, target_opset)
upgraded_path = onnx_path.replace(".onnx", f"_opset{target_opset}.onnx")
onnx.save(model, upgraded_path)
return upgraded_path
except Exception as e:
print(f" ONNX opset upgrade skipped: {e}")
return onnx_path
def build_with_python_api(onnx_path, engine_path, fp16=True, int8=False, fp8=False, bf16=False, per_layer_norm=False, strongly_typed=False):
"""Build engine using TensorRT Python API (no trtexec needed)."""
if not ensure_tensorrt_package():
return False
trt = _import_tensorrt()
# Upgrade ONNX opset to 17+ for native INormalizationLayer (fixes FP16 LayerNorm overflow)
onnx_path = _upgrade_onnx_opset(onnx_path, target_opset=17)
logger = trt.Logger(trt.Logger.WARNING)
builder = trt.Builder(logger)
# TRT 11: EXPLICIT_BATCH was removed (explicit batch is the only mode) and
# STRONGLY_TYPED is the default + only typing mode. Precision comes entirely
# from the ONNX Q/DQ nodes β TRT reads each tensor's data_type as-is, with no
# backward type-inference that would propagate INT8 onto a DequantizeLinear's
# shared scale tensor. Per-precision BuilderFlags, the INT8 calibrator, and
# per-layer precision were ALL removed in TRT 11; this is now the only path.
network_flags = 1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED)
network = builder.create_network(network_flags)
print(" STRONGLY_TYPED: enabled (TRT 11 default; precision from Q/DQ ONNX nodes)")
parser = trt.OnnxParser(network, logger)
print(f" Parsing ONNX: {onnx_path}")
# Use parse_from_file if available (TRT 8.6+) β resolves external data (model.onnx_data)
# relative to the ONNX file directory. Falls back to parse(bytes) + chdir for older TRT.
parsed_ok = False
if hasattr(parser, "parse_from_file"):
parsed_ok = parser.parse_from_file(onnx_path)
else:
# Older TRT: chdir to model directory so external data is found
prev_cwd = os.getcwd()
os.chdir(os.path.dirname(os.path.abspath(onnx_path)))
try:
with open(onnx_path, "rb") as f:
parsed_ok = parser.parse(f.read())
finally:
os.chdir(prev_cwd)
if not parsed_ok:
for i in range(parser.num_errors):
print(f" ONNX parse error: {parser.get_error(i)}", file=sys.stderr)
return False
config = builder.create_builder_config()
# Workspace pool β bigger = wider tactic search. Const TRT_WORKSPACE_GB
# at module top. Default 4 GB; bumping to 8-12 GB widens tactic search.
# Build-time only β freed when builder exits.
config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, TRT_WORKSPACE_GB << 30)
print(f" Workspace pool: {TRT_WORKSPACE_GB} GB" + (" (default)" if TRT_WORKSPACE_GB == 4 else " (probe value)"))
# Max builder optimization level for best engine quality
config.builder_optimization_level = 5
# Detailed profiling info β needed for the engine_inspector dump below
# to enumerate fused layers (MHA, GEMM, SkipLayerNormalization, etc).
# Negligible build-time overhead; engine itself is unchanged.
try:
config.profiling_verbosity = trt.ProfilingVerbosity.DETAILED
except AttributeError:
pass
# 2026-04-28 β sm_120 aux-stream regression fix (researcher rec #4).
# TRT 10.13.3 / 10.16.1 release notes document a multi-stream
# codegen bug on consumer Blackwell (sm_120). Setting max_aux_streams
# to 0 bypasses the buggy path. Property added in TRT 8.6+; older
# versions silently ignore (try/except for safety).
#
# 2026-05-19 β value from module-top const TRT_MAX_AUX_STREAMS.
try:
config.max_aux_streams = TRT_MAX_AUX_STREAMS
if TRT_MAX_AUX_STREAMS == 0:
print(" max_aux_streams: 0 (sm_120 regression workaround)")
else:
print(f" max_aux_streams: {TRT_MAX_AUX_STREAMS} (probe value β TRT may auto-disable on linear graphs)")
except AttributeError:
pass
# Precision: TRT 11 reads every tensor type from the strongly-typed network
# (the ONNX Q/DQ nodes). The per-precision BuilderFlags (INT8/FP16/BF16/FP8),
# the INT8 calibrator (IInt8EntropyCalibrator2 + int8_calibrator +
# set_calibration_profile), and per-layer precision (layer.precision /
# set_output_type) were ALL removed in TRT 11 β there are no builder flags to
# set here. The INT8 W8A16 / FP8 E4M3 / NVFP4 path is selected purely by which
# *_qdt.onnx was produced by quantize_onnx.py.
print(" Precision: from ONNX Q/DQ nodes (strongly-typed; no builder flags)")
# Timing cache: speeds up rebuilds dramatically
timing_cache_path = os.path.join(os.path.dirname(engine_path), "timing.cache")
if os.path.exists(timing_cache_path):
with open(timing_cache_path, "rb") as f:
cache = config.create_timing_cache(f.read())
print(f" Timing cache: loaded ({os.path.getsize(timing_cache_path)} bytes)")
else:
cache = config.create_timing_cache(b"")
config.set_timing_cache(cache, ignore_mismatch=False)
# Create optimization profiles
for i, (max_b, max_s, opt_b, opt_s) in enumerate(PROFILES):
profile = builder.create_optimization_profile()
# RECTANGULAR_PROFILE: min == opt == max. TRT autotuner picks a
# single set of tactics for the exact (batch, seq) shape β no
# variable-batch search, no autotuner regression on sm_120.
# Trapezoid mode: min=(1,1) β opt β max. Lets TRT accept smaller
# real batches at runtime, but autotuner must cover the whole
# range and on consumer Blackwell that range produces broken
# kernels for the upper half of the batch dimension.
if DYNSEQ_MIN_SEQ is not None:
# Batch stays rectangular (the sm_120 batch-range codegen problem
# above is untouched); only seq becomes a range.
min_b, min_s = max_b, DYNSEQ_MIN_SEQ
elif RECTANGULAR_PROFILE:
min_b, min_s = max_b, max_s
else:
min_b, min_s = 1, 1
profile.set_shape("input_ids", (min_b, min_s), (opt_b, opt_s), (max_b, max_s))
profile.set_shape("attention_mask", (min_b, min_s), (opt_b, opt_s), (max_b, max_s))
# token_type_ids if present in the model
for j in range(network.num_inputs):
inp = network.get_input(j)
if inp.name == "token_type_ids":
profile.set_shape("token_type_ids", (min_b, min_s), (opt_b, opt_s), (max_b, max_s))
break
config.add_optimization_profile(profile)
shape_kind = "RECTANGULAR" if RECTANGULAR_PROFILE else "trapezoid"
print(f" Profile {i}: {shape_kind} {min_b}x{min_s} β {opt_b}x{opt_s} β {max_b}x{max_s}")
print(f" Building engine (this may take several minutes)...")
serialized = builder.build_serialized_network(network, config)
if serialized is None:
print("ERROR: Engine build failed", file=sys.stderr)
return False
os.makedirs(os.path.dirname(engine_path), exist_ok=True)
with open(engine_path, "wb") as f:
f.write(serialized)
# Save timing cache for future rebuilds
with open(timing_cache_path, "wb") as f:
f.write(cache.serialize())
print(f" Timing cache saved: {timing_cache_path}")
# Layer-info sidecar (for MHA fusion verification, autotuner tactic
# inspection). Deserialize the just-saved engine, dump full JSON of
# all layers via engine_inspector. Needs profiling_verbosity=DETAILED
# at build time (set above) for tactic names + dtypes to be present.
layers_json_path = engine_path.replace(".engine", ".layers.json")
try:
runtime = trt.Runtime(trt.Logger(trt.Logger.WARNING))
eng = runtime.deserialize_cuda_engine(serialized)
if eng is not None:
inspector = eng.create_engine_inspector()
layer_json = inspector.get_engine_information(trt.LayerInformationFormat.JSON)
with open(layers_json_path, "w", encoding="utf-8") as fj:
fj.write(layer_json)
n_layers = layer_json.count('"LayerType"')
n_int8 = layer_json.count('"Int8"')
n_fp8 = layer_json.count('"Fp8"') + layer_json.count('"FP8"')
n_bf16 = layer_json.count('"BFloat16"') + layer_json.count('"BF16"')
# Half and Float were missing from this verdict, so an all-FP32 engine and
# a fully-FP16 one printed the identical "Int8=0 BF16=0 FP8=0" line. That
# blind spot is how the FP16 lane shipped FP32 engines unnoticed β the one
# report that could have caught it did not look at the two types involved.
n_half = layer_json.count('"Half"')
n_float = layer_json.count('"Float"')
# TRT 11's Myelin names the fused kernel `_gemm_mha_v2_*`; the two older
# spellings never appear in a TRT 11 dump, so counting only them printed
# NOT DETECTED for every engine on 2026-09-11 - including all eleven
# XLM-R engines that carry 24 of these kernels each.
lj = layer_json.lower()
n_fmha = lj.count('fmha') + lj.count('fused_multi_head_attention') + lj.count('_gemm_mha')
print(f" Layer info: {layers_json_path}")
print(f" {n_layers} layers β Half={n_half} Float={n_float} "
f"Int8={n_int8} BF16={n_bf16} FP8={n_fp8}")
if n_half == 0 and n_int8 == 0 and n_bf16 == 0 and n_fp8 == 0 and n_float > 0:
print(" WARNING: engine is ENTIRELY FP32 β no reduced precision took "
"effect. TRT 11 reads precision from the ONNX types only.")
if n_fmha > 0:
print(f" MHA fusion: DETECTED ({n_fmha} fmha tactic refs)")
else:
print(f" MHA fusion: NOT DETECTED β consider ONNX-level attention export")
except Exception as e:
print(f" Layer-info dump SKIPPED: {e}")
return True
def main():
parser = argparse.ArgumentParser(description="Build TRT engine from ONNX")
parser.add_argument(
"--model-dir",
default="dist/models/multilingual-e5-small",
help="Path to model directory containing model.onnx",
)
parser.add_argument("--cuda-arch", default=None, help="CUDA arch (e.g. sm_120)")
parser.add_argument("--fp16", action="store_true", default=True, help="Enable FP16")
parser.add_argument("--int8", action="store_true", default=True, help="Enable INT8 mixed precision")
parser.add_argument("--no-int8", action="store_true", help="Disable INT8 (FP16 only)")
parser.add_argument("--fp8", action="store_true", help="Enable FP8 (E4M3) β Hopper/Blackwell only. Implies --no-int8.")
parser.add_argument("--bf16", action="store_true", help="Enable BF16 (FP32-range half-storage) β required for Gemma RMSNorm. Disables FP16/INT8/FP8.")
parser.add_argument("--per-layer-norm", action="store_true",
help="Per-layer precision: norm layers β BF16, rest β INT8. "
"Requires --int8. Bypasses trtexec (Python API only). "
"Experimental fix for Gemma RMSNorm NaN under INT8.")
parser.add_argument("--force", action="store_true", help="Rebuild even if engine exists")
parser.add_argument("--reset-timing-cache", action="store_true",
help="Delete existing timing.cache before build. Use when switching "
"from single-profile to multi-profile builds: the cached kernel "
"selection for profile 0 shape from a single-profile build is "
"suboptimal (693 emb/s) vs a fresh multi-profile build (3800 emb/s).")
parser.add_argument("--onnx", default=None, help="Override ONNX path (bypasses find_onnx priority order)")
parser.add_argument(
"--role", default="embedding", choices=["embedding", "ner", "reranker"],
help="Model role β selects optimization profile set. "
"ner: batchβ€32, seqβ€128 (doc chunks). "
"embedding: batchβ€256, seqβ€128 (default). "
"reranker: same as embedding.",
)
parser.add_argument("--model-id", default=None,
help="Model ID for engine filename (default: basename of --model-dir)")
parser.add_argument("--os-arch", default=None,
help="OS+arch tag for engine filename (win_x64/linux_x64/linux_arm64/mac_arm64). Auto-detected if omitted.")
parser.add_argument("--variant", default=None,
help="Length-aware bucket variant name (embed: s/m/l; relation: s/m/l/xl). "
"When set, collapses PROFILES to a rectangular (max_batch, max_seq) and "
"appends `-{variant}` to the engine filename. The shape is taken from "
"--variant-batch/--variant-seq when provided (config-driven, per-model "
"from semantic_config.bucket_variants), else falls back to the embed-only "
"BUCKET_VARIANT_PROFILES table. Enables cross-worker bucketed routing.")
parser.add_argument("--variant-batch", type=int, default=None,
help="Explicit rectangular batch for --variant (overrides BUCKET_VARIANT_PROFILES).")
parser.add_argument("--variant-seq", type=int, default=None,
help="Explicit rectangular seq for --variant (overrides BUCKET_VARIANT_PROFILES).")
parser.add_argument("--no-dynseq", action="store_true",
help="Bake a rectangular sequence dimension instead of the "
"bucket's dynamic range (BUCKET_VARIANT_DYNSEQ). Costs "
"about 10%% of embedding throughput; for A/B controls.")
args = parser.parse_args()
# Select profile set based on model role before any build function is called.
# build_with_trtexec and build_with_python_api both read the global PROFILES.
global PROFILES, DYNSEQ_MIN_SEQ
if args.role == "ner":
PROFILES = NER_PROFILES
elif args.role == "reranker":
PROFILES = RERANKER_PROFILES
# --variant takes precedence over the role-default PROFILES. Bucketed variants
# apply to embedding AND reranker (relation-CE length bucketing, 2026-07-10);
# rejected for `--role ner` (NER is single-shape). The shape is config-driven:
# --variant-batch/--variant-seq (forwarded by models/compile.cmd from the model's
# semantic_config.bucket_variants) win; else fall back to the embed-only table.
if args.variant is not None:
if args.role not in ("embedding", "reranker"):
print(
f"ERROR: --variant {args.variant!r} is incompatible with --role {args.role!r}; "
f"bucketed variants are for embedding/reranker only.",
file=sys.stderr,
)
sys.exit(1)
if args.variant_batch is not None and args.variant_seq is not None:
_vb, _vs = args.variant_batch, args.variant_seq
elif args.variant in BUCKET_VARIANT_PROFILES:
_vb, _vs = BUCKET_VARIANT_PROFILES[args.variant]
else:
print(
f"ERROR: --variant {args.variant!r} has no shape: pass --variant-batch/--variant-seq "
f"or use a known embed variant {list(BUCKET_VARIANT_PROFILES.keys())}.",
file=sys.stderr,
)
sys.exit(1)
PROFILES = [(_vb, _vs, _vb, _vs)]
print(f"[build_trt] --variant {args.variant} β rectangular ({_vb}, {_vs})", flush=True)
# Dynamic sequence for the embed grid. Keyed on the SHAPE, not the bucket
# name: relation-CE reuses s/m/l/xl at seq 64/128/256/320 and its opt
# points are not these. A shape that does not match the embed table
# stays rectangular.
_dyn = BUCKET_VARIANT_DYNSEQ.get(args.variant)
if _dyn and not args.no_dynseq and \
BUCKET_VARIANT_PROFILES.get(args.variant) == (_vb, _vs):
DYNSEQ_MIN_SEQ, _opt = _dyn
PROFILES = [(_vb, _vs, _vb, _opt)]
print(f"[build_trt] --variant {args.variant} β dynamic seq "
f"{DYNSEQ_MIN_SEQ}β{_opt}β{_vs} at batch {_vb}", flush=True)
# Dynamic-seq probe β applied last so it wins over both the rectangular
# --variant shape and the BUCKET_VARIANT_DYNSEQ table. This is the knob for
# sweeping a candidate grid; production builds get their shape from the
# table and leave it unset.
_dynseq = os.environ.get("DYNSEQ_OVERRIDE", "").strip()
if _dynseq:
try:
_db, _dmin, _dopt, _dmax = (int(x) for x in _dynseq.split(","))
PROFILES = [(_db, _dmax, _db, _dopt)]
DYNSEQ_MIN_SEQ = _dmin
print(
f"[build_trt] DYNSEQ probe: batch {_db} rectangular, seq {_dmin}β{_dopt}β{_dmax}",
flush=True,
)
except (ValueError, AttributeError) as _e:
print(f"[build_trt] WARN: bad DYNSEQ_OVERRIDE='{_dynseq}': {_e}", flush=True)
use_bf16 = args.bf16
use_fp8 = args.fp8 and not use_bf16
use_int8 = args.int8 and not args.no_int8 and not use_fp8 and not use_bf16
use_fp16 = args.fp16 and not use_bf16
use_per_layer_norm = args.per_layer_norm and use_int8 and not use_bf16
use_strongly_typed = False
cuda_arch = args.cuda_arch or get_cuda_arch()
if not cuda_arch:
print("ERROR: Cannot detect CUDA arch. Pass --cuda-arch sm_XXX", file=sys.stderr)
sys.exit(1)
# `--cuda-arch` names the FILE; TensorRT always builds for the GPU that is
# present. So the detected architecture is the ONLY one this machine may
# produce, and a request for any other is refused rather than honoured with
# a rename. There is no override: an engine carrying an architecture it was
# not built for fails `deserializeEngine` on the real card with no fallback,
# and it fails there, on a user's machine, not here. Every other arch is
# built on a host that has it (Selectel VMs for sm_75/86/89).
#
# This is not hypothetical: dist/ held 30 engines across four models named
# sm_75/sm_86/sm_89 that were byte-identical to their sm_120 twins.
_detected = get_cuda_arch()
if _detected and args.cuda_arch and args.cuda_arch != _detected:
print(f"ERROR: --cuda-arch {args.cuda_arch} but this machine is "
f"{_detected}. TensorRT cannot build for an absent architecture; "
f"the engine would be a mislabelled {_detected} plan. Build it on "
f"a {args.cuda_arch} host.", file=sys.stderr)
sys.exit(2)
if args.onnx:
onnx_path = args.onnx
if not os.path.exists(onnx_path):
print(f"ERROR: --onnx path not found: {onnx_path}", file=sys.stderr)
sys.exit(1)
else:
# Mode-aware ONNX discovery: FP8 build β prefer model_fp8qdt.onnx;
# INT8 β model_int8qdt.onnx; BF16 β plain model.onnx.
# Without this, --fp8 mode would pick up model_int8qdt.onnx whose
# Q/DQ nodes are INT8-typed, defeating the FP8 builder flag and
# silently producing an INT8 engine.
_discover_mode = "fp8" if use_fp8 else ("bf16" if use_bf16 else "int8")
onnx_path = find_onnx(args.model_dir, mode=_discover_mode)
if onnx_path is not None and use_fp8 and not onnx_is_fp8(onnx_path):
# FP8 requested but no model_fp8qdt.onnx β find_onnx fell back
# to INT8 Q/DQ ONNX. Per line 1124 logic this auto-enables
# both --int8 and --fp8 builder flags, but TRT will only run
# INT8 (no FP8 Q/DQ nodes in the file). The resulting engine
# is INT8 disguised as FP8. Warn loudly so the user can run
# `quantize_onnx.py --fp8` first.
print(f"WARNING: --fp8 requested but {os.path.basename(onnx_path)} is INT8 Q/DQ.")
print(f"WARNING: Resulting engine will be INT8 (no FP8 Tensor Cores).")
print(f"WARNING: Run `scripts/quantize_onnx.py --fp8 --model-dir {args.model_dir}` first.")
if not onnx_path:
print(f"ERROR: ONNX model not found in {args.model_dir}", file=sys.stderr)
sys.exit(1)
# β MAKE THE FP16 LANE ACTUALLY FP16.
#
# TRT 11 is STRONGLY-TYPED ONLY: the per-precision builder flags were removed and
# precision is read entirely from the ONNX tensor types. `catalog.py` routes
# rerankers and `trt_fp16_only` models here with `--no-int8` and labels the build
# FP16, but nothing converted the graph β so every engine on this lane shipped as
# FP32. Measured 2026-08-03 across the deployed set: relation-v1 Float=335 Half=0,
# reranker-v1 Float=325 Half=0, denoise-v1 Float=331 Half=0 (its catalog entry even
# says BF16). Only embed-v1 was genuinely quantized, because its ONNX carries Q/DQ
# nodes and strong typing therefore has something to read.
#
# Converting and rebuilding: relation 20.74 -> 5.44 ms/chunk (3.81x), reranker
# 21.14 -> 5.40 (3.91x), denoise 8.56 -> 3.60 (2.38x), each verified against the
# previously deployed engine at cos >= 0.999995 with argmax fully preserved.
# Engines halve on disk too. It is ~4x rather than the ~1.4x a DRAM argument
# predicts because FP32 matmuls do not use the tensor cores at all.
#
# The tag drives it, as it always did β this only makes the flag mean its label.
# Re-verify a converted model with:
# models/_compile/tools/build_fp16_engine.py --stage verify --model <name>
if not use_int8 and not use_fp8 and not use_bf16:
try:
sys.path.insert(0, os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(
os.path.abspath(__file__)))), "tools"))
from onnx_fp16 import ensure_fp16_onnx
onnx_path = ensure_fp16_onnx(onnx_path)
except Exception as e: # noqa: BLE001 - never block a build on the conversion
print(f"WARNING: FP16 ONNX conversion failed ({e}); "
f"building FP32 from {os.path.basename(onnx_path)}")
model_id = args.model_id or os.path.basename(os.path.normpath(args.model_dir))
if args.os_arch:
os_arch = args.os_arch
else:
import platform as _platform
_machine = _platform.machine().lower()
_is_arm = _machine in ("arm64", "aarch64")
if sys.platform == "win32":
os_arch = "win_arm64" if _is_arm else "win_x64"
elif sys.platform == "darwin":
os_arch = "mac_arm64"
else:
os_arch = "linux_arm64" if _is_arm else "linux_x64"
# Length-aware variant suffix slotted into filename BEFORE the runtime
# tag β keeps the established `..._{os_arch}_trt{VER}_{cuda_arch}.engine`
# cluster intact so existing CDN tooling continues to glob by suffix.
# Layout: `{model_id}-{variant}_{os_arch}_trt{VER}_{cuda_arch}.engine`.
variant_tag = f"-{args.variant}" if args.variant else ""
rt_tag = _trt_runtime_tag()
_refuse_trt_version_skew(rt_tag)
engine_path = os.path.join(args.model_dir, f"{model_id}{variant_tag}_{os_arch}_{rt_tag}_{cuda_arch}.engine")
if os.path.exists(engine_path) and not args.force:
size_mb = os.path.getsize(engine_path) / (1024 * 1024)
print(f"Engine already exists: {engine_path} ({size_mb:.1f} MB)")
print("Use --force to rebuild")
sys.exit(0)
if args.reset_timing_cache:
timing_cache_path = os.path.join(args.model_dir, "timing.cache")
if os.path.exists(timing_cache_path):
os.remove(timing_cache_path)
print(f" Timing cache deleted: {timing_cache_path} (fresh kernel selection)")
# Explicit Q/DQ ONNX detected: override precision mode based on Q/DQ flavour.
# INT8 Q/DQ path β STRONGLY_TYPED: prevents TRT's backward type-inference from propagating
# INT8 from DequantizeLinear's input to the shared Q/DQ scale initializer.
# ORT's quantize_static creates one scale initializer shared by both Q and DQ in each pair.
# In weakly-typed mode this causes "[SCALE] has invalid precision Int8, ignored" and disables
# W8A16 fusion (verified 363 warnings = 363 DQ nodes, TRT 10.16, sm_120).
# With STRONGLY_TYPED, TRT reads precision from ONNX data_type; no builder flags needed.
#
# FP8 Q/DQ path β also STRONGLY_TYPED (corrected 2026-05-19): the prior
# "FP8 STRONGLY_TYPED = 171 emb/s" datum was measured on a plain `model.onnx`
# WITHOUT Q/DQ nodes β TRT fell through to the generic codepath. With a
# real FP8 Q/DQ ONNX from ModelOpt, STRONGLY_TYPED reads FP8 precision +
# BF16 scale initializers directly from the graph, avoiding the
# weakly-typed validate-error trap: `bf16 precision has been set ... but
# bf16 is not configured in the builder` (TRT 10.16 silently rejects
# `config.set_flag(BF16)` when also FP8+INT8+FP16 are set, sm_120).
# No precision builder flags are set in this mode β TRT honours the
# ONNX types as-is.
#
# INT8 Q/DQ ONNX + --fp8 (legacy mixed path) β keep weakly-typed
# FP16+INT8+FP8: meaningful only when caller mixes an INT8 Q/DQ ONNX with
# --fp8 hint. Currently no production model hits this branch; preserved
# for backward compatibility.
if onnx_has_qdq(onnx_path):
use_bf16 = False
use_per_layer_norm = False
if onnx_is_fp8(onnx_path):
# FP8 Q/DQ ONNX β STRONGLY_TYPED reads everything from graph.
use_fp16 = False
use_int8 = False
use_fp8 = False # builder flags suppressed; FP8 comes from Q/DQ nodes
use_strongly_typed = True
print(f"[PTQ] Explicit FP8 Q/DQ ONNX β STRONGLY_TYPED (FP8 Tensor Cores + BF16 scales from ONNX)")
elif use_fp8:
# INT8 Q/DQ ONNX + --fp8: weakly-typed mixed path (legacy).
use_fp16 = True
use_int8 = True
use_strongly_typed = False
print(f"[PTQ] INT8 Q/DQ ONNX + --fp8 β FP16+INT8+FP8 (kFP8 Tensor Cores for Q/DQ layers)")
else:
# INT8 Q/DQ ONNX, current production path.
use_fp16 = False
use_int8 = False
use_strongly_typed = True
print(f"[PTQ] Explicit Q/DQ ONNX β STRONGLY_TYPED (W8A16 path, no [SCALE] type-inference warnings)")
if use_strongly_typed:
# Differentiate between INT8 Q/DQ (W8A16) and FP8 Q/DQ (E4M3+BF16).
# `onnx_is_fp8` is filename-based and was already evaluated before
# the flag-resolution block above.
precision = "STRONGLY_TYPED+FP8" if onnx_is_fp8(onnx_path) else "STRONGLY_TYPED+W8A16"
elif use_bf16:
precision = "BF16"
elif use_fp8:
precision = "FP8+FP16"
elif use_int8 and use_per_layer_norm:
precision = "INT8+BF16-norm"
elif use_int8:
precision = "FP16+INT8"
else:
precision = "FP16"
print(f"Building TRT engine: {cuda_arch} ({precision})")
print(f" ONNX: {onnx_path}")
print(f" Engine: {engine_path}")
print()
ok = False
# Per-layer-norm mode requires Python API (trtexec has no per-layer precision CLI).
# FP8 (weakly-typed kFP8+kFP16) and all other modes can use trtexec.
trtexec = None
if not use_per_layer_norm:
trtexec = shutil.which("trtexec")
if not trtexec:
for p in [
"/usr/bin/trtexec",
"/usr/local/bin/trtexec",
]:
if os.path.isfile(p) and os.access(p, os.X_OK):
trtexec = p
break
if not trtexec:
import glob
for pattern in [
"/usr/local/TensorRT-*/bin/trtexec",
"/opt/tensorrt/bin/trtexec",
"/usr/local/cuda/bin/trtexec",
]:
matches = glob.glob(pattern)
if matches:
trtexec = matches[-1] # latest version
break
if trtexec:
print(f"Using trtexec: {trtexec}")
ok = build_with_trtexec(onnx_path, engine_path, trtexec_bin=trtexec)
else:
print("trtexec not in PATH β using Python TensorRT API")
ok = build_with_python_api(onnx_path, engine_path)
# TRT 11 has a SINGLE build path: strongly-typed, precision read from the
# ONNX Q/DQ nodes. The TRT-10 fallback cascade (retry as weakly-typed
# FP16+INT8 / BF16 / FP8 / calibrated-INT8) was removed along with those
# builder flags + the calibrator β there is nothing weaker to fall back to.
# If the strongly-typed build fails, it fails; the caller surfaces the TRT
# error from the log above.
if not ok or not os.path.exists(engine_path):
print("ERROR: Engine build failed", file=sys.stderr)
sys.exit(1)
size_mb = os.path.getsize(engine_path) / (1024 * 1024)
print(f"\nSUCCESS: {engine_path} ({size_mb:.1f} MB)")
if __name__ == "__main__":
main()
|