File size: 10,210 Bytes
40da26e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
#!/usr/bin/env bash
# bootstrap_linux.sh - turn a bare Linux GPU VM into a linux_x64 TRT engine builder.
#
# The Linux twin of bootstrap_win.ps1, and the fallback when a Windows VM's WSL
# cannot see the GPU (Tesla cards in TCC mode, vGPU profiles, nested virt off -
# bootstrap_win diagnoses which). Everything it needs comes over HTTPS: the tools
# from TOOLS_URL, the ONNX from the public HF model repos, and the engines go back
# to those same repos. No S3 credentials touch the box.
#
# ── On-VM usage (the only command you type) ───────────────────────────────────────────
#   export TOOLS_URL=https://<bucket>.s3.twcstorage.ru/_build-tools
#   export HF_TOKEN=hf_...write...
#   curl -fsSL "$TOOLS_URL/bootstrap_linux.sh" | bash
#
# Optional:
#   DEST_PREFIX   also copy the engines to S3 (needs S3_* creds too)
#   DISTRO_PY     python to use (default: python3)
#   ONLY / SKIP   space-separated model ids, passed through to run_matrix.py
set -uo pipefail

C='\033[0;36m'; G='\033[0;32m'; R='\033[0;31m'; Y='\033[0;33m'; D='\033[0;90m'; Z='\033[0m'
head_() { echo; echo -e "${D}$(printf '=%.0s' {1..72})${Z}"; echo -e "${C} $*${Z}"; echo -e "${D}$(printf '=%.0s' {1..72})${Z}"; }
step()  { echo -e "\n${C}[>] $*${Z}"; }
ok()    { echo -e "  ${G}[OK]${Z} $*"; }
warn()  { echo -e "  ${Y}[!] ${Z} $*"; }
die()   { echo -e "  ${R}[!!]${Z} $*" >&2; exit 1; }

TOOLS_URL="${TOOLS_URL:-}"
HF_TOKEN="${HF_TOKEN:-}"
DEST_PREFIX="${DEST_PREFIX:-}"
PY="${DISTRO_PY:-python3}"
WORK="${WORKDIR:-$HOME/uc-build}"
TOOLS="$WORK/_tools"

# build_matrix.json is what makes the run self-describing: precision per model per
# arch comes from the catalog, not from whatever the operator remembered to export.
TOOL_FILES="run_matrix.py build_matrix.json build_engines.sh build_trt_engine.py onnx_fp16.py quantize_onnx.py calib_corpus.jsonl"
# fp16-only archs never touch the quantizer or the corpus - a 404 there must not
# block a Turing run. onnx_fp16.py is NOT optional: it is a local repo module the
# worker imports inside a try, so a missing one skips the FP16 pass silently and
# ships an FP32 engine under an FP16 name.
TOOL_OPTIONAL="quantize_onnx.py calib_corpus.jsonl"

head_ "ultracode cloud builder - Linux VM bootstrap"

[ -n "$TOOLS_URL" ] || die "set TOOLS_URL first, e.g. export TOOLS_URL=https://<bucket>.s3.twcstorage.ru/_build-tools"
if [ -z "$HF_TOKEN" ] && [ -z "$DEST_PREFIX" ]; then
    die "set HF_TOKEN (a WRITE token) or DEST_PREFIX - otherwise the engines are built and discarded."
fi

# ── 0. preflight ──────────────────────────────────────────────────────────────────────
step "Preflight"
[ "$(id -u)" = "0" ] && SUDO="" || SUDO="sudo"
if [ -n "$SUDO" ] && ! command -v sudo >/dev/null 2>&1; then
    die "not root and no sudo - run as root or install sudo."
fi

# Install the driver rather than demanding one. The Terraform lane has
# cloud-init's engine-build.sh do this before calling us, so the ordering never
# showed - but this script is also the MANUAL entry point, and there it ran on a
# machine whose image carries no driver and simply refused to continue.
if ! command -v nvidia-smi >/dev/null 2>&1; then
    if ! lspci 2>/dev/null | grep -qi nvidia; then
        die "no NVIDIA device on the PCI bus - this is not a GPU instance."
    fi
    step "NVIDIA driver not present - installing (ubuntu-drivers --gpgpu)"
    export DEBIAN_FRONTEND=noninteractive
    $SUDO apt-get update -qq
    $SUDO apt-get install -y -qq ubuntu-drivers-common pciutils >/dev/null
    # β›” The fallback pin must satisfy the CUDA the TensorRT WHEELS link against,
    # not merely boot the card. Our pin is a cu13 wheel (TensorRT 11.x), and
    # `build_engines.sh` picks cu12 vs cu13 from the CUDA version the DRIVER
    # reports. A 570 driver reports CUDA 12.x, so the pinned cu13 install would
    # never be attempted: the script would ask for a cu12 wheel at our pin, fail,
    # retry UNPINNED, and the version guard in build_trt_engine.py would then
    # kill the run after the VM was already paid for. 595-server reports CUDA 13.2
    # and is what `ubuntu-drivers --gpgpu` selects on Ubuntu 24.04 today
    # (measured on an A5000, 2026-09-18).
    $SUDO ubuntu-drivers install --gpgpu || $SUDO apt-get install -y -qq nvidia-driver-595-server

    # ubuntu-drivers --gpgpu installs the kernel module and the compute libs but
    # NOT nvidia-utils, which is the package that owns /usr/bin/nvidia-smi. The
    # module then loads fine and the GPU works, while every arch-detection step
    # here and in build_engines.sh keys off nvidia-smi and reads the machine as
    # having no GPU. Measured 2026-08-28: lsmod showed nvidia/nvidia_uvm loaded
    # and `nvidia-smi` was command-not-found.
    BRANCH="$($SUDO dpkg -l 2>/dev/null | grep -o 'nvidia-compute-utils-[0-9]*-server' | head -1 | sed 's/nvidia-compute-utils-//')"
    if [ -n "$BRANCH" ]; then
        $SUDO apt-get install -y -qq "nvidia-utils-$BRANCH" >/dev/null 2>&1             || warn "could not install nvidia-utils-$BRANCH - nvidia-smi may be missing"
    fi

    if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1; then
        ok "driver active without a reboot"
    else
        echo
        warn "Driver installed, but the kernel module is not live until a REBOOT."
        warn "Reboot, then run this script again - it will skip straight to the build:"
        warn "    reboot"
        warn "    curl -fsSL \"\$TOOLS_URL/bootstrap_linux.sh\" | bash"
        exit 2
    fi
fi
CC="$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d ' ')"
GPU="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | sed 's/^ *//')"
DRV="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1 | tr -d ' ')"
[ -n "$CC" ] || die "nvidia-smi reported no compute capability - driver present but GPU unusable."
SM="sm_${CC//./}"
ok "GPU: $GPU   compute_cap $CC -> $SM   driver $DRV"

. /etc/os-release 2>/dev/null || true
ok "OS: ${PRETTY_NAME:-unknown}"

# ── 1. system packages ────────────────────────────────────────────────────────────────
# python3 + pip + curl only. TensorRT arrives as a pip wheel from
# build_trt_engine.py and CUDA comes from the driver - installing a CUDA toolkit
# would only add a second, possibly mismatched, libcuda to the search path.
step "System packages"
if command -v apt-get >/dev/null 2>&1; then
    export DEBIAN_FRONTEND=noninteractive
    $SUDO apt-get update -qq || warn "apt-get update failed - continuing with what is installed"
    $SUDO apt-get install -y -qq python3 python3-pip python3-venv curl ca-certificates >/dev/null \
        || warn "apt-get install returned non-zero - continuing"
elif command -v dnf >/dev/null 2>&1; then
    $SUDO dnf install -y -q python3 python3-pip curl ca-certificates >/dev/null || warn "dnf install returned non-zero"
elif command -v yum >/dev/null 2>&1; then
    $SUDO yum install -y -q python3 python3-pip curl ca-certificates >/dev/null || warn "yum install returned non-zero"
else
    warn "unknown package manager - assuming python3/pip/curl are already present"
fi

command -v "$PY" >/dev/null 2>&1 || die "$PY not found after package install."
ok "python: $($PY --version 2>&1)"

# PEP 668 marks distro pythons as externally managed; --break-system-packages is
# the sanctioned override and is exactly right on a throwaway build box.
PIPFLAGS=""
$PY -m pip install --dry-run --break-system-packages -q pip >/dev/null 2>&1 && PIPFLAGS="--break-system-packages"
$PY -m pip install $PIPFLAGS -q -U pip huggingface_hub >/dev/null 2>&1 \
    || warn "pip bootstrap returned non-zero - run_matrix will retry per-package"
ok "pip ready${PIPFLAGS:+ ($PIPFLAGS)}"

# ── 2. fetch the build tools ──────────────────────────────────────────────────────────
step "Fetching build tools from $TOOLS_URL"
mkdir -p "$TOOLS"
cd "$TOOLS"
GOT=""; MISSING=""
for f in $TOOL_FILES; do
    if curl -fsSL -o "$f" "${TOOLS_URL%/}/$f"; then
        GOT="$GOT $f"
    else
        case " $TOOL_OPTIONAL " in
            *" $f "*) MISSING="$MISSING $f" ;;
            *) die "could not fetch $f from $TOOLS_URL - is the prefix public?" ;;
        esac
    fi
done
ok "fetched:$GOT"
[ -n "$MISSING" ] && warn "absent (fine unless an INT8 arch needs PTQ):$MISSING"
chmod +x build_engines.sh 2>/dev/null || true

# ── 3. build ──────────────────────────────────────────────────────────────────────────
head_ "Linux lane - building linux_x64 engines on $SM"
ARGS=""
for m in ${ONLY:-}; do ARGS="$ARGS --only $m"; done
for m in ${SKIP:-}; do ARGS="$ARGS --skip $m"; done

TOOLS_URL="$TOOLS_URL" HF_TOKEN="$HF_TOKEN" DEST_PREFIX="$DEST_PREFIX" WORKDIR="$WORK" \
    $PY run_matrix.py $ARGS
RC=$?

# ── 4. report ─────────────────────────────────────────────────────────────────────────
head_ "RESULT  -  $GPU / $SM"
if [ "$RC" = "0" ]; then
    echo -e "  linux_x64  : ${G}OK${Z}   (all models)"
else
    echo -e "  linux_x64  : ${R}$RC model(s) FAILED${Z}"
fi
echo
echo -e "${D}  Verify what actually landed:${Z}"
echo -e "${D}    https://huggingface.co/faxenoff/code-daemon-embed-v1/tree/main${Z}"
echo -e "${D}    every name must end in _${SM}.engine and contain _linux_x64 for this box${Z}"
echo
echo -e "${D}  The matching win_x64 cells come from bootstrap_win.ps1 on a Windows GPU VM${Z}"
echo -e "${D}  with the same card - an engine built here will NOT load on Windows.${Z}"
exit $RC