File size: 60,475 Bytes
40da26e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
#!/usr/bin/env python3
"""Build TensorRT engine from ONNX model with multi-profile bucketing.

Usage:
    python scripts/build_trt_engine.py [--model-dir PATH] [--cuda-arch sm_120]

Two modes:
  1. trtexec (if in PATH) β€” NVIDIA's official CLI tool
  2. Python TensorRT API (fallback) β€” uses tensorrt pip package

Profiles (3 buckets matching tensorrt_runtime.zig default_profiles):
  0: batch 1-32,  seq 1-32   (opt: 16x32)   β€” short entity names
  1: batch 1-64,  seq 1-64   (opt: 32x64)   β€” entity descriptions
  2: batch 1-128, seq 1-128  (opt: 128x128) β€” docstrings / function bodies

Output: {model_id}_{os_arch}_trt{VER}_{cuda_arch}.engine in the model directory
        (os_arch = win_x64 / linux_x64 / linux_arm64 / mac_arm64;
         VER = TensorRT major.minor from vendor/tensorrt/VERSION, e.g. trt11.0).

A serialized plan is keyed on {GPU arch x OS x TRT version} and TensorRT offers
no compat fallback on deserialize, so the version belongs in the filename next
to the arch: the daemon then resolves an engine it can actually load instead of
opening a same-named plan from another TRT and failing inside the runtime.
"""

import argparse
import os
import shutil
import subprocess
import sys
import warnings
from pathlib import Path

# Force UTF-8 output on Windows (default console encoding is CP1251/CP1252)
if hasattr(sys.stdout, "reconfigure"):
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
if hasattr(sys.stderr, "reconfigure"):
    sys.stderr.reconfigure(encoding="utf-8", errors="replace")

# Auto-install pip deps on first run when this script is invoked directly
# (i.e. without going through models/code-daemon-embed/run.ps1 / run.sh which
# already install everything). Idempotent β€” skips packages that are already importable.
if os.environ.get("ULTRACODE_NO_BOOTSTRAP") != "1":
    # Repo root by walking up to build.zig, not by counting `..` β€” this file has
    # moved once already (scripts/ -> models/_compile/backends/workers/) and a
    # hardcoded depth breaks silently, since a missing directory just means "no
    # bootstrap available".
    _root = Path(__file__).resolve()
    for _ in range(6):
        _root = _root.parent
        if (_root / "build.zig").is_file():
            break
    _bootstrap_dir = _root / "models" / "code-daemon-embed"
    if (_bootstrap_dir / "_bootstrap.py").exists():
        sys.path.insert(0, str(_bootstrap_dir))
        try:
            from _bootstrap import ensure_deps, ensure_nvidia_deps
            # onnx is needed for ensure_single_file_onnx() and proto inspection;
            # torch is not required by this script β€” skip the torch install.
            ensure_deps(extra=["onnx"], torch_required=False)
            # tensorrt + cuda-python (NVIDIA-only). No-op on macOS / CPU hosts.
            ensure_nvidia_deps()
        except Exception as _bs_err:
            print(f"[bootstrap] WARNING: {_bs_err}", flush=True)
        finally:
            sys.path.remove(str(_bootstrap_dir))

# Suppress TRT deprecation warnings: implicit INT8 calibration (superseded by QAT/explicit
# quantization in TRT 10.1+) and per-layer precision (superseded by strong typing in TRT 10.12+).
# Replacements require ONNX QAT nodes / STRONGLY_TYPED mode β€” not applicable here.
warnings.filterwarnings("ignore", message=".*Deprecated.*", category=DeprecationWarning)
warnings.filterwarnings("ignore", message=".*deprecated.*", category=DeprecationWarning)


def ensure_single_file_onnx(onnx_path):
    """Convert split ONNX (model.onnx + model.onnx_data) to single-file if needed."""
    data_file = onnx_path + "_data"
    if not os.path.exists(data_file):
        return onnx_path

    merged_path = onnx_path.replace(".onnx", "_merged.onnx")
    if os.path.exists(merged_path):
        merged_size = os.path.getsize(merged_path)
        if merged_size < 1024 or merged_size > 2_000_000_000:
            # Empty/corrupted or >2GB (protobuf can't parse) β€” delete stale cache
            os.remove(merged_path)
            print(f"  Removed stale merged ONNX ({merged_size / 1e6:.0f} MB)")
        else:
            merged_mtime = os.path.getmtime(merged_path)
            if (merged_mtime > os.path.getmtime(onnx_path) and
                merged_mtime > os.path.getmtime(data_file)):
                print(f"  Split ONNX: using cached merged file")
                return merged_path

    try:
        import onnx
        print(f"  Split ONNX detected: merging into single file...")
        model = onnx.load(onnx_path, load_external_data=True)
        onnx.save(model, merged_path)
        merged_size = os.path.getsize(merged_path)
        if merged_size > 2_000_000_000:
            # >2GB: protobuf can't parse it back, trtexec may fail. Use original + external data.
            os.remove(merged_path)
            print(f"  Merged {merged_size / 1e6:.0f} MB > 2GB limit, using split ONNX (parse_from_file handles external data)")
            return onnx_path
        print(f"  Merged: {merged_size / (1024*1024):.1f} MB")
        return merged_path
    except Exception as e:
        print(f"  Split ONNX merge failed: {e}")
        return onnx_path

# Profile specs: (max_batch, max_seq, opt_batch, opt_seq)
#
# Single rectangular profile per model type β€” no multi-profile bucketing.
# The daemon accumulates up to batch=128 before dispatch; buildEmbedInput
# caps at 420 chars β‰ˆ 64 tokens so seq=64 covers all real traffic.
# Fewer profiles = less VRAM, faster compile, trivial selectProfile (always 0).
# Multi-profile builds were tried and produce slower kernels for the seq=64 path
# on Gemma sm_120 (autotuner picks kernels optimized for the broader shape range).
#
# Profile specs: (max_batch, max_seq, opt_batch, opt_seq).
#
# 2026-05-05 β€” half-batch-zero saga resolved. The "TRT sm_120 regression"
# we chased turned out to be a Zig-side dtype mismatch: engine input
# binding is INT64 (verified via getTensorDtype), but our worker copied
# host buffers as INT32 stride. Engine read 8 bytes per token from a
# buffer where we'd written 4 β†’ got 2Γ— rows of garbage from uninit
# memory, manifesting as "second half of every batch is zero". Fixed
# in tensorrt_runtime.zig + embed_worker.zig (i32 β†’ i64 throughout the
# TRT input chain). Engine itself is fine; restore full 128 batch.
PROFILES = [
    (128, 64, 128, 64),  # 0: rectangular fast path β€” batch 128, seq 64
]
RECTANGULAR_PROFILE = True

# Dynamic-seq probe (2026-07-28). Set `DYNSEQ_OVERRIDE="batch,min_seq,opt_seq,max_seq"`
# to build ONE engine whose SEQUENCE dimension is a range while the batch stays
# rectangular. Purpose: a chunk could then be dispatched at its own longest text
# instead of the bucket's ceiling β€” live telemetry says 15.3% of GPU token-work
# is padding (avg 42/48, 56/64, 70/80, 95/128). The question this answers is
# whether TRT's broader-range kernel choice costs more than the padding it saves;
# the file header already records that multi-PROFILE builds regressed seq=64 on
# sm_120. Applied AFTER --variant in main() so the engine still gets a suffixed
# filename. Production builds leave it unset.
DYNSEQ_MIN_SEQ = None

# ─── Experimental switches (hardcoded) ──────────────────────────────────
# Project preference: in-source const flip is cleaner than env-var dance
# for one-off experiments. Each switch documents its evidence + revert.
# Flip the value, save, run models/compile or build_trt_engine directly.

# Builder workspace pool. Default 4 GB; bump to 8-12 GB to widen TRT
# tactic search on autotuning. Build-time only β€” does NOT affect engine
# size or runtime VRAM. Risk: build OOM if value > free VRAM at build time.
# REVERT: change back to 4.
TRT_WORKSPACE_GB = 8

# TRT max_aux_streams config.
#   0 β€” workaround for the 2026-04-28 "5 valid + N zeros" sm_120 codegen
#       bug (TRT 10.13.3). Forces single-stream kernel generation.
#   4 β€” TRT default. Researcher confirmed 2026-05-19 that post-fusion
#       embedding graphs (BGE/Qwen3/Gemma) are too linear for aux streams
#       to help, and TRT 10.16 gracefully self-disables ("Multi stream is
#       disabled because the stream assignment failed" warning). Either
#       value produces functionally identical engines on this family.
# Kept as probe knob β€” REVERT: 0.
TRT_MAX_AUX_STREAMS = 0

# Env override for sweep experiments. Format: RECT_PROFILE_OVERRIDE="batch,seq"
# β€” collapses PROFILES to a single (batch, seq, batch, seq) entry. Production
# builds leave this unset. Example: RECT_PROFILE_OVERRIDE="32,32".
_rect_override = os.environ.get("RECT_PROFILE_OVERRIDE", "").strip()
if _rect_override:
    try:
        _b, _s = (int(x) for x in _rect_override.split(","))
        PROFILES = [(_b, _s, _b, _s)]
        print(f"[build_trt] PROFILES overridden via env: rectangular ({_b}, {_s})", flush=True)
    except (ValueError, AttributeError) as _e:
        print(f"[build_trt] WARN: bad RECT_PROFILE_OVERRIDE='{_rect_override}': {_e}", flush=True)

# Length-aware bucket variants β€” when `--variant {s,m,l}` is passed, PROFILES
# collapses to that variant's rectangular (max_batch, max_seq). Single source
# of truth lives in `models/code-daemon-embed/embed_bucket_config.py`. We
# duplicate the tuples here to avoid forcing the build to import from a model
# subdirectory at parse-time (the bootstrap path may not yet be on sys.path).
# `models/_compile/preflight_bucket_sync.py` runs as a pre-build gate to catch drift.
BUCKET_VARIANT_PROFILES = {
    # 2026-07-27: grid 40/64/128 β†’ 32/48/80/128, measured on real engines
    # (see the header comment in src/inference/embed_bucket_config.zig).
    "s": (96, 48),
    "m": (128, 64),
    "l": (128, 80),
    "xl": (256, 128),
}

# (min_seq, opt_seq) per embed bucket β€” the SEQUENCE dimension is a range while
# the batch stays rectangular, so a chunk is dispatched at its own longest text
# instead of the bucket ceiling. Live telemetry put 15.3% of GPU token-work in
# sequence padding; recovering it measured +10.2% end to end (9,738 -> 10,731
# emb/s on mysql-server) and is quality-neutral by construction β€” padding is
# masked in attention, verified at cosine 1.000000 on 128 rows of real ids.
#
# Two rules produced these numbers and neither is guessable:
#   - `opt` sits at the bucket's histogram-weighted MEAN, not its ceiling. The
#     tax is paid for being far from the opt point, not for being dynamic: one
#     wide 16β†’64β†’128 engine costs +0.8% at seq 64 and +15.4% at seq 80.
#   - `min` is the bucket FLOOR, not the observed corpus minimum β€” `s` gets 8
#     rather than the measured 24, because the router must accept a 3-token text.
#
# This lived in a DYNSEQ_OVERRIDE env var and was deployed by copying files over
# the rectangular ones. A later `models/compile` run rebuilt them rectangular and
# the +10.2% was silently gone; the engines still loaded, still returned 768-dim
# vectors, and reported 54 layers instead of 69 where nobody was looking. Hence
# the table: a plain build now produces the shape that is meant to ship.
BUCKET_VARIANT_DYNSEQ = {
    "s": (8, 48),
    "m": (56, 56),
    "l": (72, 72),
    "xl": (88, 96),
}

# NER single profile: batch=32 (NER worker --batch-size 32), seq=128.
# Bumped from seq=64 (2026-05-05) β€” ~20% of incoming chunks exceed 64
# tokens (long entity descriptions, function bodies, docstrings), causing
# `SetShapeFailed` at runtime and DocGraph falling back to rules-only for
# those entities. The OV NER variants ship as `_b32_s128`, so the ONNX
# already supports seq=128. Opt point sits at 96 β€” middle of the 1..128
# range, biasing the autotuner toward the typical mass of inputs without
# regressing the long tail.
NER_PROFILES = [
    # 0: NER main path β€” RECTANGULAR (max == opt) to avoid an invalid
    # min<=opt<=max profile. The previous value (32, 128, 32, 96) set
    # opt_seq=96 < max_seq=128, which under RECTANGULAR_PROFILE (min==max==128)
    # makes opt_seq < min_seq β†’ TRT 11 rejects with "Invalid optimization
    # profiles ... ensure MIN<=OPT<=MAX". The NER worker always pads to
    # batch=32 seq=128, so a static 32x128 profile is correct.
    (32, 128, 32, 128),
]

# Reranker single profile: batch=16 (reranker worker max_batch=16, matches the OV engine
# `_b16_s256` and models/compile.cmd's $ovChkBatch=16), seq=256. The cross-encoder scores
# (query, document) PAIRS, which are long, so seq must be 256 β€” NOT the embedding default
# (128, 64). Without a reranker-specific profile, `--role reranker` fell through to the
# embedding PROFILES (128, 64); the engine bound seq=64 while handleRerank pads to
# max_seq_len=256 β†’ SetShapeFailed at runtime β†’ search degraded to no-rerank. RECTANGULAR
# (min==opt==max) per handleRerank's rectangular-engine contract (embed_worker.zig:4867).
RERANKER_PROFILES = [
    (16, 256, 16, 256),
]


def _installed_trt_tag():
    """`trt{major}.{minor}` of the TensorRT in THIS interpreter, or None."""
    try:
        import tensorrt as trt
        parts = str(trt.__version__).split(".")
    except Exception:
        return None
    return f"trt{parts[0]}.{parts[1]}" if len(parts) >= 2 and parts[0] else None


def _pinned_trt_tag():
    """`trt{major}.{minor}` from `vendor/tensorrt/VERSION`, or None if no repo.

    This is the pin build.zig turns into the daemon's `runtime_versions.trt_tag`,
    so naming from it guarantees the engine we write is the engine the worker
    looks for. Absent when the script runs standalone β€” the cloud sm_86/sm_89
    builder (models/_cloud_shared/build-tools/build_engines.sh) copies only this
    .py onto a bare VM, with no vendor/ tree to read.
    """
    d = os.path.dirname(os.path.abspath(__file__))
    for _ in range(6):  # walk up to repo root
        pin = os.path.join(d, "vendor", "tensorrt", "VERSION")
        if os.path.isfile(pin):
            try:
                parts = open(pin).read().strip().split(".")
            except OSError:
                return None
            return f"trt{parts[0]}.{parts[1]}" if len(parts) >= 2 and parts[0] else None
        parent = os.path.dirname(d)
        if parent == d:
            return None
        d = parent
    return None


def _trt_runtime_tag():
    """Runtime-version tag embedded in every engine filename, e.g. `trt11.0`.

    Prefers the repo pin (what the DAEMON will deserialize with); falls back to
    the TensorRT actually doing the build, so a standalone/cloud run still
    produces a correctly-labelled engine instead of failing. Only when neither is
    knowable do we stop: an untagged engine is invisible to the daemon, and TRT
    builds are far too slow to discover that afterwards.
    """
    tag = _pinned_trt_tag() or _installed_trt_tag()
    if tag:
        return tag
    print("ERROR: cannot determine the TensorRT version β€” neither vendor/tensorrt/VERSION")
    print("       nor an importable `tensorrt` package. Engine filenames carry the")
    print("       TensorRT major.minor and the daemon only loads its own version.")
    sys.exit(1)


def _refuse_trt_version_skew(rt_tag):
    """Stop when the TensorRT building this engine is not the pinned one.

    Matters more here than anywhere else: the plan is produced by the `tensorrt`
    package in THIS interpreter but deserialized by the vendored nvinfer the
    daemon links. If they differ, the file gets a name promising a runtime that
    cannot read it β€” a load failure the version tag exists to prevent.

    This was a WARNING until 2026-09-12. It printed on every Linux build for
    weeks while every Linux engine was built by 11.2 and named trt11.0
    (`models/_compile/deps/wsl.py:_trt_pin`), because nothing consumed it. There
    is no legitimate skew: a patch-level difference yields the same tag and never
    reaches this branch, and a standalone cloud run has no pin, so `rt_tag` IS the
    installed tag there. A missing package is handled elsewhere.
    """
    installed_tag = _installed_trt_tag()
    if installed_tag and installed_tag != rt_tag:
        print(f"ERROR: TensorRT version skew β€” this interpreter has {installed_tag} "
              f"but the engine would be named {rt_tag} (from vendor/tensorrt/VERSION).")
        print(f"       The daemon deserializes with the pinned runtime and would "
              f"REJECT the plan. Install tensorrt-cu13 at the pin, or bump the pin.")
        sys.exit(1)


def get_cuda_arch():
    """Detect CUDA compute capability via nvidia-smi."""
    try:
        out = subprocess.check_output(
            ["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"],
            text=True,
        ).strip().split("\n")[0]
        major, minor = out.split(".")
        return f"sm_{major}{minor}"
    except Exception:
        return None


def _onnx_is_valid(path):
    """Quick-check that an ONNX file is parseable (not truncated).
    Files >1 MB that fail onnx.load are assumed valid: Python protobuf has a 2 GB
    parse limit but trtexec uses the C++ ONNX parser which has no such limit."""
    try:
        import onnx
        onnx.load(path, load_external_data=False)
        return True
    except Exception as e:
        try:
            size = os.path.getsize(path)
            if size > 1024 * 1024:
                print(f"  NOTE: onnx.load failed ({e}), file is {size / (1024*1024):.0f} MB "
                      f"β€” assuming valid (trtexec uses C++ parser, no 2 GB limit)")
                return True
        except Exception:
            pass
        return False


def _has_all_external_data(onnx_path):
    """Return True if every external-data file referenced by the ONNX header exists
    in the same directory.  Returns True (optimistic) when the header itself cannot
    be loaded (e.g. 2 GB+ merged file β€” no external refs to check)."""
    try:
        import onnx as _onnx
        header = _onnx.load(onnx_path, load_external_data=False)
    except Exception:
        return True  # can't check β€” assume ok (2 GB+ merged, no external refs)
    onnx_dir = os.path.dirname(os.path.abspath(onnx_path))
    for t in header.graph.initializer:
        if t.data_location == _onnx.TensorProto.EXTERNAL:
            for entry in t.external_data:
                if entry.key == "location":
                    if not os.path.exists(os.path.join(onnx_dir, entry.value)):
                        return False
    return True


def find_onnx(model_dir, mode="int8"):
    """Find ONNX model in model directory, with validation and cache fallback.

    `mode` controls preference order:
      - "fp8": prefer `model_fp8qdt.onnx` (ModelOpt FP8 PTQ output) over INT8.
      - "int8" (default): prefer `model_int8qdt.onnx` (ModelOpt INT8 PTQ output).
      - "bf16": prefer plain `model.onnx`.
    Falls back through the alternate-precision PTQ ONNX before raw `model.onnx`
    so that a misconfigured catalog still produces a build (TRT will pick up
    whatever Q/DQ nodes are in the file).
    """
    # Last-resort fallback: the compiler's download cache for third-party models
    # (`models/_compile/.cache/models/<id>/`). The compiler itself always passes
    # `--onnx` explicitly and the cloud builder stages the ONNX into --model-dir,
    # so this is only reached by a hand-run invocation.
    #
    # It used to be derived as `<this script's dir>/.cache/<id>`, which pointed
    # at `scripts/.cache` while the script lived there and became a path that can
    # never exist once it moved.
    model_name = os.path.basename(os.path.normpath(model_dir))
    _pkg_dir = os.path.dirname(os.path.abspath(__file__))          # …/backends/workers
    _compile_root = os.path.dirname(os.path.dirname(_pkg_dir))     # …/models/_compile
    cache_dir = os.environ.get("ULTRACODE_COMPILE_CACHE") or \
        os.path.join(_compile_root, ".cache")
    cache_path = os.path.join(cache_dir, "models", model_name, "model.onnx")

    int8_candidates = [
        os.path.join(model_dir, "onnx", "model_int8qdt.onnx"),
        os.path.join(model_dir, "model_int8qdt.onnx"),
        os.path.join(cache_dir, "models", model_name, "model_int8qdt.onnx"),
    ]
    fp8_candidates = [
        os.path.join(model_dir, "onnx", "model_fp8qdt.onnx"),
        os.path.join(model_dir, "model_fp8qdt.onnx"),
        os.path.join(cache_dir, "models", model_name, "model_fp8qdt.onnx"),
    ]
    plain_candidates = [
        os.path.join(model_dir, "model.onnx"),
        os.path.join(model_dir, "onnx", "model.onnx"),
        cache_path,
    ]

    if mode == "fp8":
        candidates = fp8_candidates + int8_candidates + plain_candidates
    elif mode == "bf16":
        candidates = plain_candidates + int8_candidates + fp8_candidates
    else:
        candidates = int8_candidates + plain_candidates + fp8_candidates

    for p in candidates:
        if os.path.exists(p):
            if not _onnx_is_valid(p):
                print(f"  WARNING: {p} is corrupted (truncated?), trying next candidate")
                continue
            if not _has_all_external_data(p):
                print(f"  NOTE: {p} has missing external data files β€” trying next candidate")
                continue
            return p
    return None


def onnx_has_qdq(onnx_path):
    """True if the ONNX model contains explicit Q/DQ nodes (PTQ quantized).

    When Q/DQ nodes are present, TRT must have a matching precision builder
    flag enabled (INT8 for int8qdt, FP8 for fp8qdt). Without it, TRT silently
    ignores DequantizeLinear/QuantizeLinear and builds a plain BF16/FP16
    engine, wasting the calibration work.
    Detection by filename is sufficient β€” quantize_onnx.py always writes
    'model_int8qdt.onnx' (INT8 mode) or 'model_fp8qdt.onnx' (FP8 mode).
    """
    name = os.path.basename(onnx_path)
    return any(tag in name for tag in ("int8qdt", "fp8qdt", "nvfp4qdt", "int4qdt"))


def onnx_is_fp8(onnx_path):
    """True if the ONNX model is a ModelOpt FP8 PTQ output (fp8qdt naming).

    Used to distinguish FP8-quantized ONNX (needs --fp8 builder flag) from
    INT8-quantized ONNX (needs --int8). When neither, fallback paths apply.
    """
    return "fp8qdt" in os.path.basename(onnx_path)


def build_with_trtexec(onnx_path, engine_path, trtexec_bin="trtexec"):
    """Build engine using trtexec CLI (TRT 11: strongly-typed only)."""
    onnx_path = ensure_single_file_onnx(onnx_path)
    cmd = [trtexec_bin, f"--onnx={onnx_path}", f"--saveEngine={engine_path}"]
    # TRT 11: precision is read entirely from the ONNX Q/DQ nodes. Strongly typed
    # is the only typing mode β€” the per-precision flags (--int8/--fp16/--bf16/--fp8)
    # and the INT8 calibrator (--calib) were removed. `--stronglyTyped` is not
    # passed: TRT 11.3 release notes list it as a flag with no effect.

    # Timing cache: speeds up rebuilds dramatically (minutes β†’ seconds)
    timing_cache = os.path.join(os.path.dirname(engine_path), "timing.cache")
    cmd.append(f"--timingCacheFile={timing_cache}")
    # Max builder optimization level for best engine quality
    cmd.append("--builderOptimizationLevel=5")
    # 2026-04-28 β€” sm_120 aux-stream regression fix (researcher rec #4 in
    # .autodoc/archive/inference-backends.md). TRT 10.13.3 / 10.16.1 release notes
    # document a multi-stream codegen bug on consumer Blackwell (sm_120)
    # that affects pooling/attention subgraphs and produces partial
    # outputs (the embeddinggemma "5 valid + N-5 zeros" symptom matches
    # this signature: kernel runs but writes zeros to trailing rows).
    # Disabling aux streams bypasses the buggy codegen path; cost is
    # one stream of dispatch concurrency we don't currently exploit.
    #
    # 2026-05-19 β€” value lives at module top as TRT_MAX_AUX_STREAMS const.
    # Flip there + rebuild to probe (researcher confirmed it's effectively
    # a no-op on embedding models; TRT 10.16 self-disables anyway).
    cmd.append(f"--maxAuxStreams={TRT_MAX_AUX_STREAMS}")
    if TRT_MAX_AUX_STREAMS != 0:
        print(f"  [build_trt] TRT_MAX_AUX_STREAMS={TRT_MAX_AUX_STREAMS} β€” sm_120 workaround DISABLED")
    # `--useCudaGraph` / `--useSpinWait` were passed here until 2026-09-22. They
    # only shaped trtexec's post-build benchmark, never the saved engine, and
    # TRT 11.3 release notes list both as flags with no effect β€” so a benchmark
    # line printed by an 11.3 trtexec is measured WITHOUT graphs and does not
    # compare with 10.16/11.0 logs. The runtime counterpart was measured and
    # left off: `cuda_graphs_enabled` in src/inference/tensorrt_runtime.zig.

    for max_b, max_s, opt_b, opt_s in PROFILES:
        cmd.extend([
            f"--minShapes=input_ids:1x1,attention_mask:1x1",
            f"--optShapes=input_ids:{opt_b}x{opt_s},attention_mask:{opt_b}x{opt_s}",
            f"--maxShapes=input_ids:{max_b}x{max_s},attention_mask:{max_b}x{max_s}",
        ])

    print(f"  trtexec command: {' '.join(cmd[:5])} ...")
    result = subprocess.run(cmd)
    return result.returncode == 0


def _add_nvinfer_to_path():
    """Add vendor/tensorrt/lib AND pip tensorrt_libs to DLL search path."""
    added = []

    # 1. vendor/tensorrt/lib (our bundled runtime DLLs).
    # This file lives at models/_compile/backends/workers/, so the repo root is
    # four levels up. It used to be one (`scripts/../vendor`), which stopped
    # resolving the moment the script moved β€” silently, because the directory
    # simply "was not there" and the pip tensorrt_libs below usually covered it.
    _here = os.path.dirname(os.path.abspath(__file__))
    _repo_root = os.path.abspath(os.path.join(_here, "..", "..", "..", ".."))
    vendor_lib = os.path.join(_repo_root, "vendor", "tensorrt", "lib")
    if os.path.isdir(vendor_lib):
        added.append(vendor_lib)

    # 2. pip tensorrt_libs (contains nvinfer_plugin_10.dll needed for builder)
    try:
        import site
        for sp in site.getsitepackages() + [site.getusersitepackages()]:
            trt_libs = os.path.join(sp, "tensorrt_libs")
            if os.path.isdir(trt_libs):
                added.append(trt_libs)
                break
    except Exception:
        pass

    # Register all paths
    for lib_dir in added:
        if sys.platform == "win32" and hasattr(os, "add_dll_directory"):
            try:
                os.add_dll_directory(lib_dir)
            except OSError:
                pass
        os.environ["PATH"] = lib_dir + os.pathsep + os.environ.get("PATH", "")

    return added[0] if added else None


def _import_tensorrt():
    """Import tensorrt, handling the cu12_bindings layout."""
    _add_nvinfer_to_path()

    # Try direct import first (tensorrt meta-package installed)
    try:
        import tensorrt
        return tensorrt
    except (ImportError, ModuleNotFoundError, FileNotFoundError, OSError):
        pass

    # Try bindings-only import (tensorrt_cu12_bindings installed)
    try:
        from tensorrt_bindings import tensorrt
        return tensorrt
    except (ImportError, ModuleNotFoundError, FileNotFoundError, OSError):
        pass

    return None


def _has_cuda_runtime():
    """Check if CUDA runtime is available (libcudart loadable)."""
    import ctypes
    for lib in ["libcudart.so", "libcudart.so.13", "libcudart.so.12", "libcudart.so.11",
                "cudart64_13.dll", "cudart64_12.dll", "cudart64_110.dll"]:
        try:
            ctypes.cdll.LoadLibrary(lib)
            return True
        except OSError:
            pass
    # Also check if nvcc is in PATH (CUDA toolkit installed)
    if shutil.which("nvcc"):
        return True
    return False


def ensure_tensorrt_package():
    """Auto-install tensorrt Python bindings + libs if missing."""
    trt = _import_tensorrt()
    if trt is not None:
        return True

    if not _has_cuda_runtime():
        print("  tensorrt not available: CUDA runtime not found (no libcudart/nvcc)", file=sys.stderr)
        print("  Install CUDA toolkit or add trtexec to PATH", file=sys.stderr)
        return False

    print("  tensorrt not found β€” installing via pip...")
    nvidia_idx = "https://pypi.nvidia.com"

    # Detect CUDA major version to pick matching tensorrt build. Mixing cu12
    # and cu13 packages produces version-mismatch errors at engine deserialize
    # because the engine plan encodes the build-time native DLL version, and
    # `import tensorrt` then resolves to a different package's DLL at runtime.
    cuda_major = None
    try:
        nvsmi = subprocess.check_output(
            ["nvidia-smi"], stderr=subprocess.DEVNULL, text=True, timeout=5,
        )
        import re
        m = re.search(r"CUDA Version:\s*(\d+)\.(\d+)", nvsmi)
        if m:
            cuda_major = int(m.group(1))
    except Exception:
        pass

    if cuda_major and cuda_major >= 13:
        candidates = [
            ["tensorrt-cu13-bindings==11.0.0.114", "tensorrt-cu13-libs==11.0.0.114"],
            ["tensorrt-cu13-bindings", "tensorrt-cu13-libs"],
        ]
    else:
        candidates = [
            ["tensorrt-cu12-bindings==11.0.0.114", "tensorrt-cu12-libs==11.0.0.114"],
            ["tensorrt-cu12-bindings", "tensorrt-cu12-libs"],
        ]

    for pkgs in candidates:
        result = subprocess.run(
            [sys.executable, "-m", "pip", "install", "--quiet",
             "--only-binary", ":all:",
             "--extra-index-url", nvidia_idx] + pkgs,
            capture_output=True, text=True,
        )
        if result.returncode == 0:
            break
    else:
        # Last resort: try the meta-package
        subprocess.run(
            [sys.executable, "-m", "pip", "install", "--quiet",
             "--only-binary", ":all:",
             "--extra-index-url", nvidia_idx,
             "tensorrt"],
            capture_output=True, text=True,
        )

    trt = _import_tensorrt()
    if trt is not None:
        print(f"  tensorrt {trt.__version__} ready")
        return True

    print("  tensorrt still not importable after install", file=sys.stderr)
    return False


def _upgrade_onnx_opset(onnx_path, target_opset=17):
    """Upgrade ONNX model to target opset if needed (fixes LayerNorm FP16 overflow)."""
    # Handle split ONNX first β€” skip upgrade if merge fails (>2GB models)
    data_file = onnx_path + "_data"
    if os.path.exists(data_file):
        merged = ensure_single_file_onnx(onnx_path)
        if merged == onnx_path:
            # Merge failed (protobuf 2GB limit) β€” skip opset upgrade, TRT will parse directly
            print(f"  Opset upgrade skipped: split ONNX too large to merge")
            return onnx_path
        onnx_path = merged
    else:
        onnx_path = ensure_single_file_onnx(onnx_path)
    try:
        import onnx
        from onnx import version_converter
        model = onnx.load(onnx_path)
        current_opset = model.opset_import[0].version if model.opset_import else 0
        if current_opset >= target_opset:
            return onnx_path  # already sufficient
        print(f"  Upgrading ONNX opset {current_opset} β†’ {target_opset} (fixes LayerNorm FP16 overflow)")
        model = version_converter.convert_version(model, target_opset)
        upgraded_path = onnx_path.replace(".onnx", f"_opset{target_opset}.onnx")
        onnx.save(model, upgraded_path)
        return upgraded_path
    except Exception as e:
        print(f"  ONNX opset upgrade skipped: {e}")
        return onnx_path


def build_with_python_api(onnx_path, engine_path, fp16=True, int8=False, fp8=False, bf16=False, per_layer_norm=False, strongly_typed=False):
    """Build engine using TensorRT Python API (no trtexec needed)."""
    if not ensure_tensorrt_package():
        return False

    trt = _import_tensorrt()

    # Upgrade ONNX opset to 17+ for native INormalizationLayer (fixes FP16 LayerNorm overflow)
    onnx_path = _upgrade_onnx_opset(onnx_path, target_opset=17)

    logger = trt.Logger(trt.Logger.WARNING)
    builder = trt.Builder(logger)

    # TRT 11: EXPLICIT_BATCH was removed (explicit batch is the only mode) and
    # STRONGLY_TYPED is the default + only typing mode. Precision comes entirely
    # from the ONNX Q/DQ nodes β€” TRT reads each tensor's data_type as-is, with no
    # backward type-inference that would propagate INT8 onto a DequantizeLinear's
    # shared scale tensor. Per-precision BuilderFlags, the INT8 calibrator, and
    # per-layer precision were ALL removed in TRT 11; this is now the only path.
    network_flags = 1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED)
    network = builder.create_network(network_flags)
    print("  STRONGLY_TYPED: enabled (TRT 11 default; precision from Q/DQ ONNX nodes)")
    parser = trt.OnnxParser(network, logger)

    print(f"  Parsing ONNX: {onnx_path}")
    # Use parse_from_file if available (TRT 8.6+) β€” resolves external data (model.onnx_data)
    # relative to the ONNX file directory. Falls back to parse(bytes) + chdir for older TRT.
    parsed_ok = False
    if hasattr(parser, "parse_from_file"):
        parsed_ok = parser.parse_from_file(onnx_path)
    else:
        # Older TRT: chdir to model directory so external data is found
        prev_cwd = os.getcwd()
        os.chdir(os.path.dirname(os.path.abspath(onnx_path)))
        try:
            with open(onnx_path, "rb") as f:
                parsed_ok = parser.parse(f.read())
        finally:
            os.chdir(prev_cwd)
    if not parsed_ok:
        for i in range(parser.num_errors):
            print(f"  ONNX parse error: {parser.get_error(i)}", file=sys.stderr)
            return False

    config = builder.create_builder_config()
    # Workspace pool β€” bigger = wider tactic search. Const TRT_WORKSPACE_GB
    # at module top. Default 4 GB; bumping to 8-12 GB widens tactic search.
    # Build-time only β€” freed when builder exits.
    config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, TRT_WORKSPACE_GB << 30)
    print(f"  Workspace pool: {TRT_WORKSPACE_GB} GB" + (" (default)" if TRT_WORKSPACE_GB == 4 else " (probe value)"))
    # Max builder optimization level for best engine quality
    config.builder_optimization_level = 5
    # Detailed profiling info β€” needed for the engine_inspector dump below
    # to enumerate fused layers (MHA, GEMM, SkipLayerNormalization, etc).
    # Negligible build-time overhead; engine itself is unchanged.
    try:
        config.profiling_verbosity = trt.ProfilingVerbosity.DETAILED
    except AttributeError:
        pass
    # 2026-04-28 β€” sm_120 aux-stream regression fix (researcher rec #4).
    # TRT 10.13.3 / 10.16.1 release notes document a multi-stream
    # codegen bug on consumer Blackwell (sm_120). Setting max_aux_streams
    # to 0 bypasses the buggy path. Property added in TRT 8.6+; older
    # versions silently ignore (try/except for safety).
    #
    # 2026-05-19 β€” value from module-top const TRT_MAX_AUX_STREAMS.
    try:
        config.max_aux_streams = TRT_MAX_AUX_STREAMS
        if TRT_MAX_AUX_STREAMS == 0:
            print("  max_aux_streams: 0 (sm_120 regression workaround)")
        else:
            print(f"  max_aux_streams: {TRT_MAX_AUX_STREAMS} (probe value β€” TRT may auto-disable on linear graphs)")
    except AttributeError:
        pass

    # Precision: TRT 11 reads every tensor type from the strongly-typed network
    # (the ONNX Q/DQ nodes). The per-precision BuilderFlags (INT8/FP16/BF16/FP8),
    # the INT8 calibrator (IInt8EntropyCalibrator2 + int8_calibrator +
    # set_calibration_profile), and per-layer precision (layer.precision /
    # set_output_type) were ALL removed in TRT 11 β€” there are no builder flags to
    # set here. The INT8 W8A16 / FP8 E4M3 / NVFP4 path is selected purely by which
    # *_qdt.onnx was produced by quantize_onnx.py.
    print("  Precision: from ONNX Q/DQ nodes (strongly-typed; no builder flags)")

    # Timing cache: speeds up rebuilds dramatically
    timing_cache_path = os.path.join(os.path.dirname(engine_path), "timing.cache")
    if os.path.exists(timing_cache_path):
        with open(timing_cache_path, "rb") as f:
            cache = config.create_timing_cache(f.read())
        print(f"  Timing cache: loaded ({os.path.getsize(timing_cache_path)} bytes)")
    else:
        cache = config.create_timing_cache(b"")
    config.set_timing_cache(cache, ignore_mismatch=False)

    # Create optimization profiles
    for i, (max_b, max_s, opt_b, opt_s) in enumerate(PROFILES):
        profile = builder.create_optimization_profile()
        # RECTANGULAR_PROFILE: min == opt == max. TRT autotuner picks a
        # single set of tactics for the exact (batch, seq) shape β€” no
        # variable-batch search, no autotuner regression on sm_120.
        # Trapezoid mode: min=(1,1) β†’ opt β†’ max. Lets TRT accept smaller
        # real batches at runtime, but autotuner must cover the whole
        # range and on consumer Blackwell that range produces broken
        # kernels for the upper half of the batch dimension.
        if DYNSEQ_MIN_SEQ is not None:
            # Batch stays rectangular (the sm_120 batch-range codegen problem
            # above is untouched); only seq becomes a range.
            min_b, min_s = max_b, DYNSEQ_MIN_SEQ
        elif RECTANGULAR_PROFILE:
            min_b, min_s = max_b, max_s
        else:
            min_b, min_s = 1, 1
        profile.set_shape("input_ids", (min_b, min_s), (opt_b, opt_s), (max_b, max_s))
        profile.set_shape("attention_mask", (min_b, min_s), (opt_b, opt_s), (max_b, max_s))
        # token_type_ids if present in the model
        for j in range(network.num_inputs):
            inp = network.get_input(j)
            if inp.name == "token_type_ids":
                profile.set_shape("token_type_ids", (min_b, min_s), (opt_b, opt_s), (max_b, max_s))
                break
        config.add_optimization_profile(profile)
        shape_kind = "RECTANGULAR" if RECTANGULAR_PROFILE else "trapezoid"
        print(f"  Profile {i}: {shape_kind} {min_b}x{min_s} β†’ {opt_b}x{opt_s} β†’ {max_b}x{max_s}")

    print(f"  Building engine (this may take several minutes)...")
    serialized = builder.build_serialized_network(network, config)
    if serialized is None:
        print("ERROR: Engine build failed", file=sys.stderr)
        return False

    os.makedirs(os.path.dirname(engine_path), exist_ok=True)
    with open(engine_path, "wb") as f:
        f.write(serialized)

    # Save timing cache for future rebuilds
    with open(timing_cache_path, "wb") as f:
        f.write(cache.serialize())
    print(f"  Timing cache saved: {timing_cache_path}")

    # Layer-info sidecar (for MHA fusion verification, autotuner tactic
    # inspection). Deserialize the just-saved engine, dump full JSON of
    # all layers via engine_inspector. Needs profiling_verbosity=DETAILED
    # at build time (set above) for tactic names + dtypes to be present.
    layers_json_path = engine_path.replace(".engine", ".layers.json")
    try:
        runtime = trt.Runtime(trt.Logger(trt.Logger.WARNING))
        eng = runtime.deserialize_cuda_engine(serialized)
        if eng is not None:
            inspector = eng.create_engine_inspector()
            layer_json = inspector.get_engine_information(trt.LayerInformationFormat.JSON)
            with open(layers_json_path, "w", encoding="utf-8") as fj:
                fj.write(layer_json)
            n_layers = layer_json.count('"LayerType"')
            n_int8 = layer_json.count('"Int8"')
            n_fp8 = layer_json.count('"Fp8"') + layer_json.count('"FP8"')
            n_bf16 = layer_json.count('"BFloat16"') + layer_json.count('"BF16"')
            # Half and Float were missing from this verdict, so an all-FP32 engine and
            # a fully-FP16 one printed the identical "Int8=0 BF16=0 FP8=0" line. That
            # blind spot is how the FP16 lane shipped FP32 engines unnoticed β€” the one
            # report that could have caught it did not look at the two types involved.
            n_half = layer_json.count('"Half"')
            n_float = layer_json.count('"Float"')
            # TRT 11's Myelin names the fused kernel `_gemm_mha_v2_*`; the two older
            # spellings never appear in a TRT 11 dump, so counting only them printed
            # NOT DETECTED for every engine on 2026-09-11 - including all eleven
            # XLM-R engines that carry 24 of these kernels each.
            lj = layer_json.lower()
            n_fmha = lj.count('fmha') + lj.count('fused_multi_head_attention') + lj.count('_gemm_mha')
            print(f"  Layer info: {layers_json_path}")
            print(f"    {n_layers} layers β€” Half={n_half} Float={n_float} "
                  f"Int8={n_int8} BF16={n_bf16} FP8={n_fp8}")
            if n_half == 0 and n_int8 == 0 and n_bf16 == 0 and n_fp8 == 0 and n_float > 0:
                print("    WARNING: engine is ENTIRELY FP32 β€” no reduced precision took "
                      "effect. TRT 11 reads precision from the ONNX types only.")
            if n_fmha > 0:
                print(f"    MHA fusion: DETECTED ({n_fmha} fmha tactic refs)")
            else:
                print(f"    MHA fusion: NOT DETECTED β€” consider ONNX-level attention export")
    except Exception as e:
        print(f"  Layer-info dump SKIPPED: {e}")

    return True


def main():
    parser = argparse.ArgumentParser(description="Build TRT engine from ONNX")
    parser.add_argument(
        "--model-dir",
        default="dist/models/multilingual-e5-small",
        help="Path to model directory containing model.onnx",
    )
    parser.add_argument("--cuda-arch", default=None, help="CUDA arch (e.g. sm_120)")
    parser.add_argument("--fp16", action="store_true", default=True, help="Enable FP16")
    parser.add_argument("--int8", action="store_true", default=True, help="Enable INT8 mixed precision")
    parser.add_argument("--no-int8", action="store_true", help="Disable INT8 (FP16 only)")
    parser.add_argument("--fp8", action="store_true", help="Enable FP8 (E4M3) β€” Hopper/Blackwell only. Implies --no-int8.")
    parser.add_argument("--bf16", action="store_true", help="Enable BF16 (FP32-range half-storage) β€” required for Gemma RMSNorm. Disables FP16/INT8/FP8.")
    parser.add_argument("--per-layer-norm", action="store_true",
                        help="Per-layer precision: norm layers β†’ BF16, rest β†’ INT8. "
                             "Requires --int8. Bypasses trtexec (Python API only). "
                             "Experimental fix for Gemma RMSNorm NaN under INT8.")
    parser.add_argument("--force", action="store_true", help="Rebuild even if engine exists")
    parser.add_argument("--reset-timing-cache", action="store_true",
                        help="Delete existing timing.cache before build. Use when switching "
                             "from single-profile to multi-profile builds: the cached kernel "
                             "selection for profile 0 shape from a single-profile build is "
                             "suboptimal (693 emb/s) vs a fresh multi-profile build (3800 emb/s).")
    parser.add_argument("--onnx", default=None, help="Override ONNX path (bypasses find_onnx priority order)")
    parser.add_argument(
        "--role", default="embedding", choices=["embedding", "ner", "reranker"],
        help="Model role β€” selects optimization profile set. "
             "ner: batch≀32, seq≀128 (doc chunks). "
             "embedding: batch≀256, seq≀128 (default). "
             "reranker: same as embedding.",
    )
    parser.add_argument("--model-id", default=None,
                        help="Model ID for engine filename (default: basename of --model-dir)")
    parser.add_argument("--os-arch", default=None,
                        help="OS+arch tag for engine filename (win_x64/linux_x64/linux_arm64/mac_arm64). Auto-detected if omitted.")
    parser.add_argument("--variant", default=None,
                        help="Length-aware bucket variant name (embed: s/m/l; relation: s/m/l/xl). "
                             "When set, collapses PROFILES to a rectangular (max_batch, max_seq) and "
                             "appends `-{variant}` to the engine filename. The shape is taken from "
                             "--variant-batch/--variant-seq when provided (config-driven, per-model "
                             "from semantic_config.bucket_variants), else falls back to the embed-only "
                             "BUCKET_VARIANT_PROFILES table. Enables cross-worker bucketed routing.")
    parser.add_argument("--variant-batch", type=int, default=None,
                        help="Explicit rectangular batch for --variant (overrides BUCKET_VARIANT_PROFILES).")
    parser.add_argument("--variant-seq", type=int, default=None,
                        help="Explicit rectangular seq for --variant (overrides BUCKET_VARIANT_PROFILES).")
    parser.add_argument("--no-dynseq", action="store_true",
                        help="Bake a rectangular sequence dimension instead of the "
                             "bucket's dynamic range (BUCKET_VARIANT_DYNSEQ). Costs "
                             "about 10%% of embedding throughput; for A/B controls.")
    args = parser.parse_args()

    # Select profile set based on model role before any build function is called.
    # build_with_trtexec and build_with_python_api both read the global PROFILES.
    global PROFILES, DYNSEQ_MIN_SEQ
    if args.role == "ner":
        PROFILES = NER_PROFILES
    elif args.role == "reranker":
        PROFILES = RERANKER_PROFILES

    # --variant takes precedence over the role-default PROFILES. Bucketed variants
    # apply to embedding AND reranker (relation-CE length bucketing, 2026-07-10);
    # rejected for `--role ner` (NER is single-shape). The shape is config-driven:
    # --variant-batch/--variant-seq (forwarded by models/compile.cmd from the model's
    # semantic_config.bucket_variants) win; else fall back to the embed-only table.
    if args.variant is not None:
        if args.role not in ("embedding", "reranker"):
            print(
                f"ERROR: --variant {args.variant!r} is incompatible with --role {args.role!r}; "
                f"bucketed variants are for embedding/reranker only.",
                file=sys.stderr,
            )
            sys.exit(1)
        if args.variant_batch is not None and args.variant_seq is not None:
            _vb, _vs = args.variant_batch, args.variant_seq
        elif args.variant in BUCKET_VARIANT_PROFILES:
            _vb, _vs = BUCKET_VARIANT_PROFILES[args.variant]
        else:
            print(
                f"ERROR: --variant {args.variant!r} has no shape: pass --variant-batch/--variant-seq "
                f"or use a known embed variant {list(BUCKET_VARIANT_PROFILES.keys())}.",
                file=sys.stderr,
            )
            sys.exit(1)
        PROFILES = [(_vb, _vs, _vb, _vs)]
        print(f"[build_trt] --variant {args.variant} β†’ rectangular ({_vb}, {_vs})", flush=True)

        # Dynamic sequence for the embed grid. Keyed on the SHAPE, not the bucket
        # name: relation-CE reuses s/m/l/xl at seq 64/128/256/320 and its opt
        # points are not these. A shape that does not match the embed table
        # stays rectangular.
        _dyn = BUCKET_VARIANT_DYNSEQ.get(args.variant)
        if _dyn and not args.no_dynseq and \
                BUCKET_VARIANT_PROFILES.get(args.variant) == (_vb, _vs):
            DYNSEQ_MIN_SEQ, _opt = _dyn
            PROFILES = [(_vb, _vs, _vb, _opt)]
            print(f"[build_trt] --variant {args.variant} β†’ dynamic seq "
                  f"{DYNSEQ_MIN_SEQ}β†’{_opt}β†’{_vs} at batch {_vb}", flush=True)

    # Dynamic-seq probe β€” applied last so it wins over both the rectangular
    # --variant shape and the BUCKET_VARIANT_DYNSEQ table. This is the knob for
    # sweeping a candidate grid; production builds get their shape from the
    # table and leave it unset.
    _dynseq = os.environ.get("DYNSEQ_OVERRIDE", "").strip()
    if _dynseq:
        try:
            _db, _dmin, _dopt, _dmax = (int(x) for x in _dynseq.split(","))
            PROFILES = [(_db, _dmax, _db, _dopt)]
            DYNSEQ_MIN_SEQ = _dmin
            print(
                f"[build_trt] DYNSEQ probe: batch {_db} rectangular, seq {_dmin}β†’{_dopt}β†’{_dmax}",
                flush=True,
            )
        except (ValueError, AttributeError) as _e:
            print(f"[build_trt] WARN: bad DYNSEQ_OVERRIDE='{_dynseq}': {_e}", flush=True)

    use_bf16 = args.bf16
    use_fp8 = args.fp8 and not use_bf16
    use_int8 = args.int8 and not args.no_int8 and not use_fp8 and not use_bf16
    use_fp16 = args.fp16 and not use_bf16
    use_per_layer_norm = args.per_layer_norm and use_int8 and not use_bf16
    use_strongly_typed = False
    cuda_arch = args.cuda_arch or get_cuda_arch()
    if not cuda_arch:
        print("ERROR: Cannot detect CUDA arch. Pass --cuda-arch sm_XXX", file=sys.stderr)
        sys.exit(1)

    # `--cuda-arch` names the FILE; TensorRT always builds for the GPU that is
    # present. So the detected architecture is the ONLY one this machine may
    # produce, and a request for any other is refused rather than honoured with
    # a rename. There is no override: an engine carrying an architecture it was
    # not built for fails `deserializeEngine` on the real card with no fallback,
    # and it fails there, on a user's machine, not here. Every other arch is
    # built on a host that has it (Selectel VMs for sm_75/86/89).
    #
    # This is not hypothetical: dist/ held 30 engines across four models named
    # sm_75/sm_86/sm_89 that were byte-identical to their sm_120 twins.
    _detected = get_cuda_arch()
    if _detected and args.cuda_arch and args.cuda_arch != _detected:
        print(f"ERROR: --cuda-arch {args.cuda_arch} but this machine is "
              f"{_detected}. TensorRT cannot build for an absent architecture; "
              f"the engine would be a mislabelled {_detected} plan. Build it on "
              f"a {args.cuda_arch} host.", file=sys.stderr)
        sys.exit(2)

    if args.onnx:
        onnx_path = args.onnx
        if not os.path.exists(onnx_path):
            print(f"ERROR: --onnx path not found: {onnx_path}", file=sys.stderr)
            sys.exit(1)
    else:
        # Mode-aware ONNX discovery: FP8 build β†’ prefer model_fp8qdt.onnx;
        # INT8 β†’ model_int8qdt.onnx; BF16 β†’ plain model.onnx.
        # Without this, --fp8 mode would pick up model_int8qdt.onnx whose
        # Q/DQ nodes are INT8-typed, defeating the FP8 builder flag and
        # silently producing an INT8 engine.
        _discover_mode = "fp8" if use_fp8 else ("bf16" if use_bf16 else "int8")
        onnx_path = find_onnx(args.model_dir, mode=_discover_mode)
        if onnx_path is not None and use_fp8 and not onnx_is_fp8(onnx_path):
            # FP8 requested but no model_fp8qdt.onnx β€” find_onnx fell back
            # to INT8 Q/DQ ONNX. Per line 1124 logic this auto-enables
            # both --int8 and --fp8 builder flags, but TRT will only run
            # INT8 (no FP8 Q/DQ nodes in the file). The resulting engine
            # is INT8 disguised as FP8. Warn loudly so the user can run
            # `quantize_onnx.py --fp8` first.
            print(f"WARNING: --fp8 requested but {os.path.basename(onnx_path)} is INT8 Q/DQ.")
            print(f"WARNING: Resulting engine will be INT8 (no FP8 Tensor Cores).")
            print(f"WARNING: Run `scripts/quantize_onnx.py --fp8 --model-dir {args.model_dir}` first.")
    if not onnx_path:
        print(f"ERROR: ONNX model not found in {args.model_dir}", file=sys.stderr)
        sys.exit(1)

    # ⭐ MAKE THE FP16 LANE ACTUALLY FP16.
    #
    # TRT 11 is STRONGLY-TYPED ONLY: the per-precision builder flags were removed and
    # precision is read entirely from the ONNX tensor types. `catalog.py` routes
    # rerankers and `trt_fp16_only` models here with `--no-int8` and labels the build
    # FP16, but nothing converted the graph β€” so every engine on this lane shipped as
    # FP32. Measured 2026-08-03 across the deployed set: relation-v1 Float=335 Half=0,
    # reranker-v1 Float=325 Half=0, denoise-v1 Float=331 Half=0 (its catalog entry even
    # says BF16). Only embed-v1 was genuinely quantized, because its ONNX carries Q/DQ
    # nodes and strong typing therefore has something to read.
    #
    # Converting and rebuilding: relation 20.74 -> 5.44 ms/chunk (3.81x), reranker
    # 21.14 -> 5.40 (3.91x), denoise 8.56 -> 3.60 (2.38x), each verified against the
    # previously deployed engine at cos >= 0.999995 with argmax fully preserved.
    # Engines halve on disk too. It is ~4x rather than the ~1.4x a DRAM argument
    # predicts because FP32 matmuls do not use the tensor cores at all.
    #
    # The tag drives it, as it always did β€” this only makes the flag mean its label.
    # Re-verify a converted model with:
    #   models/_compile/tools/build_fp16_engine.py --stage verify --model <name>
    if not use_int8 and not use_fp8 and not use_bf16:
        try:
            sys.path.insert(0, os.path.join(
                os.path.dirname(os.path.dirname(os.path.dirname(
                    os.path.abspath(__file__)))), "tools"))
            from onnx_fp16 import ensure_fp16_onnx
            onnx_path = ensure_fp16_onnx(onnx_path)
        except Exception as e:  # noqa: BLE001 - never block a build on the conversion
            print(f"WARNING: FP16 ONNX conversion failed ({e}); "
                  f"building FP32 from {os.path.basename(onnx_path)}")

    model_id = args.model_id or os.path.basename(os.path.normpath(args.model_dir))
    if args.os_arch:
        os_arch = args.os_arch
    else:
        import platform as _platform
        _machine = _platform.machine().lower()
        _is_arm = _machine in ("arm64", "aarch64")
        if sys.platform == "win32":
            os_arch = "win_arm64" if _is_arm else "win_x64"
        elif sys.platform == "darwin":
            os_arch = "mac_arm64"
        else:
            os_arch = "linux_arm64" if _is_arm else "linux_x64"
    # Length-aware variant suffix slotted into filename BEFORE the runtime
    # tag β€” keeps the established `..._{os_arch}_trt{VER}_{cuda_arch}.engine`
    # cluster intact so existing CDN tooling continues to glob by suffix.
    # Layout: `{model_id}-{variant}_{os_arch}_trt{VER}_{cuda_arch}.engine`.
    variant_tag = f"-{args.variant}" if args.variant else ""
    rt_tag = _trt_runtime_tag()
    _refuse_trt_version_skew(rt_tag)
    engine_path = os.path.join(args.model_dir, f"{model_id}{variant_tag}_{os_arch}_{rt_tag}_{cuda_arch}.engine")

    if os.path.exists(engine_path) and not args.force:
        size_mb = os.path.getsize(engine_path) / (1024 * 1024)
        print(f"Engine already exists: {engine_path} ({size_mb:.1f} MB)")
        print("Use --force to rebuild")
        sys.exit(0)

    if args.reset_timing_cache:
        timing_cache_path = os.path.join(args.model_dir, "timing.cache")
        if os.path.exists(timing_cache_path):
            os.remove(timing_cache_path)
            print(f"  Timing cache deleted: {timing_cache_path} (fresh kernel selection)")

    # Explicit Q/DQ ONNX detected: override precision mode based on Q/DQ flavour.
    # INT8 Q/DQ path β†’ STRONGLY_TYPED: prevents TRT's backward type-inference from propagating
    # INT8 from DequantizeLinear's input to the shared Q/DQ scale initializer.
    # ORT's quantize_static creates one scale initializer shared by both Q and DQ in each pair.
    # In weakly-typed mode this causes "[SCALE] has invalid precision Int8, ignored" and disables
    # W8A16 fusion (verified 363 warnings = 363 DQ nodes, TRT 10.16, sm_120).
    # With STRONGLY_TYPED, TRT reads precision from ONNX data_type; no builder flags needed.
    #
    # FP8 Q/DQ path β†’ also STRONGLY_TYPED (corrected 2026-05-19): the prior
    # "FP8 STRONGLY_TYPED = 171 emb/s" datum was measured on a plain `model.onnx`
    # WITHOUT Q/DQ nodes β€” TRT fell through to the generic codepath. With a
    # real FP8 Q/DQ ONNX from ModelOpt, STRONGLY_TYPED reads FP8 precision +
    # BF16 scale initializers directly from the graph, avoiding the
    # weakly-typed validate-error trap: `bf16 precision has been set ... but
    # bf16 is not configured in the builder` (TRT 10.16 silently rejects
    # `config.set_flag(BF16)` when also FP8+INT8+FP16 are set, sm_120).
    # No precision builder flags are set in this mode β€” TRT honours the
    # ONNX types as-is.
    #
    # INT8 Q/DQ ONNX + --fp8 (legacy mixed path) β†’ keep weakly-typed
    # FP16+INT8+FP8: meaningful only when caller mixes an INT8 Q/DQ ONNX with
    # --fp8 hint. Currently no production model hits this branch; preserved
    # for backward compatibility.
    if onnx_has_qdq(onnx_path):
        use_bf16 = False
        use_per_layer_norm = False
        if onnx_is_fp8(onnx_path):
            # FP8 Q/DQ ONNX β€” STRONGLY_TYPED reads everything from graph.
            use_fp16 = False
            use_int8 = False
            use_fp8 = False  # builder flags suppressed; FP8 comes from Q/DQ nodes
            use_strongly_typed = True
            print(f"[PTQ] Explicit FP8 Q/DQ ONNX β†’ STRONGLY_TYPED (FP8 Tensor Cores + BF16 scales from ONNX)")
        elif use_fp8:
            # INT8 Q/DQ ONNX + --fp8: weakly-typed mixed path (legacy).
            use_fp16 = True
            use_int8 = True
            use_strongly_typed = False
            print(f"[PTQ] INT8 Q/DQ ONNX + --fp8 β†’ FP16+INT8+FP8 (kFP8 Tensor Cores for Q/DQ layers)")
        else:
            # INT8 Q/DQ ONNX, current production path.
            use_fp16 = False
            use_int8 = False
            use_strongly_typed = True
            print(f"[PTQ] Explicit Q/DQ ONNX β€” STRONGLY_TYPED (W8A16 path, no [SCALE] type-inference warnings)")

    if use_strongly_typed:
        # Differentiate between INT8 Q/DQ (W8A16) and FP8 Q/DQ (E4M3+BF16).
        # `onnx_is_fp8` is filename-based and was already evaluated before
        # the flag-resolution block above.
        precision = "STRONGLY_TYPED+FP8" if onnx_is_fp8(onnx_path) else "STRONGLY_TYPED+W8A16"
    elif use_bf16:
        precision = "BF16"
    elif use_fp8:
        precision = "FP8+FP16"
    elif use_int8 and use_per_layer_norm:
        precision = "INT8+BF16-norm"
    elif use_int8:
        precision = "FP16+INT8"
    else:
        precision = "FP16"
    print(f"Building TRT engine: {cuda_arch} ({precision})")
    print(f"  ONNX:   {onnx_path}")
    print(f"  Engine: {engine_path}")
    print()

    ok = False

    # Per-layer-norm mode requires Python API (trtexec has no per-layer precision CLI).
    # FP8 (weakly-typed kFP8+kFP16) and all other modes can use trtexec.
    trtexec = None
    if not use_per_layer_norm:
        trtexec = shutil.which("trtexec")
        if not trtexec:
            for p in [
                "/usr/bin/trtexec",
                "/usr/local/bin/trtexec",
            ]:
                if os.path.isfile(p) and os.access(p, os.X_OK):
                    trtexec = p
                    break
        if not trtexec:
            import glob
            for pattern in [
                "/usr/local/TensorRT-*/bin/trtexec",
                "/opt/tensorrt/bin/trtexec",
                "/usr/local/cuda/bin/trtexec",
            ]:
                matches = glob.glob(pattern)
                if matches:
                    trtexec = matches[-1]  # latest version
                    break

    if trtexec:
        print(f"Using trtexec: {trtexec}")
        ok = build_with_trtexec(onnx_path, engine_path, trtexec_bin=trtexec)
    else:
        print("trtexec not in PATH β€” using Python TensorRT API")
        ok = build_with_python_api(onnx_path, engine_path)

    # TRT 11 has a SINGLE build path: strongly-typed, precision read from the
    # ONNX Q/DQ nodes. The TRT-10 fallback cascade (retry as weakly-typed
    # FP16+INT8 / BF16 / FP8 / calibrated-INT8) was removed along with those
    # builder flags + the calibrator β€” there is nothing weaker to fall back to.
    # If the strongly-typed build fails, it fails; the caller surfaces the TRT
    # error from the log above.
    if not ok or not os.path.exists(engine_path):
        print("ERROR: Engine build failed", file=sys.stderr)
        sys.exit(1)

    size_mb = os.path.getsize(engine_path) / (1024 * 1024)
    print(f"\nSUCCESS: {engine_path} ({size_mb:.1f} MB)")


if __name__ == "__main__":
    main()