File size: 10,210 Bytes
40da26e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 | #!/usr/bin/env bash
# bootstrap_linux.sh - turn a bare Linux GPU VM into a linux_x64 TRT engine builder.
#
# The Linux twin of bootstrap_win.ps1, and the fallback when a Windows VM's WSL
# cannot see the GPU (Tesla cards in TCC mode, vGPU profiles, nested virt off -
# bootstrap_win diagnoses which). Everything it needs comes over HTTPS: the tools
# from TOOLS_URL, the ONNX from the public HF model repos, and the engines go back
# to those same repos. No S3 credentials touch the box.
#
# ββ On-VM usage (the only command you type) βββββββββββββββββββββββββββββββββββββββββββ
# export TOOLS_URL=https://<bucket>.s3.twcstorage.ru/_build-tools
# export HF_TOKEN=hf_...write...
# curl -fsSL "$TOOLS_URL/bootstrap_linux.sh" | bash
#
# Optional:
# DEST_PREFIX also copy the engines to S3 (needs S3_* creds too)
# DISTRO_PY python to use (default: python3)
# ONLY / SKIP space-separated model ids, passed through to run_matrix.py
set -uo pipefail
C='\033[0;36m'; G='\033[0;32m'; R='\033[0;31m'; Y='\033[0;33m'; D='\033[0;90m'; Z='\033[0m'
head_() { echo; echo -e "${D}$(printf '=%.0s' {1..72})${Z}"; echo -e "${C} $*${Z}"; echo -e "${D}$(printf '=%.0s' {1..72})${Z}"; }
step() { echo -e "\n${C}[>] $*${Z}"; }
ok() { echo -e " ${G}[OK]${Z} $*"; }
warn() { echo -e " ${Y}[!] ${Z} $*"; }
die() { echo -e " ${R}[!!]${Z} $*" >&2; exit 1; }
TOOLS_URL="${TOOLS_URL:-}"
HF_TOKEN="${HF_TOKEN:-}"
DEST_PREFIX="${DEST_PREFIX:-}"
PY="${DISTRO_PY:-python3}"
WORK="${WORKDIR:-$HOME/uc-build}"
TOOLS="$WORK/_tools"
# build_matrix.json is what makes the run self-describing: precision per model per
# arch comes from the catalog, not from whatever the operator remembered to export.
TOOL_FILES="run_matrix.py build_matrix.json build_engines.sh build_trt_engine.py onnx_fp16.py quantize_onnx.py calib_corpus.jsonl"
# fp16-only archs never touch the quantizer or the corpus - a 404 there must not
# block a Turing run. onnx_fp16.py is NOT optional: it is a local repo module the
# worker imports inside a try, so a missing one skips the FP16 pass silently and
# ships an FP32 engine under an FP16 name.
TOOL_OPTIONAL="quantize_onnx.py calib_corpus.jsonl"
head_ "ultracode cloud builder - Linux VM bootstrap"
[ -n "$TOOLS_URL" ] || die "set TOOLS_URL first, e.g. export TOOLS_URL=https://<bucket>.s3.twcstorage.ru/_build-tools"
if [ -z "$HF_TOKEN" ] && [ -z "$DEST_PREFIX" ]; then
die "set HF_TOKEN (a WRITE token) or DEST_PREFIX - otherwise the engines are built and discarded."
fi
# ββ 0. preflight ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
step "Preflight"
[ "$(id -u)" = "0" ] && SUDO="" || SUDO="sudo"
if [ -n "$SUDO" ] && ! command -v sudo >/dev/null 2>&1; then
die "not root and no sudo - run as root or install sudo."
fi
# Install the driver rather than demanding one. The Terraform lane has
# cloud-init's engine-build.sh do this before calling us, so the ordering never
# showed - but this script is also the MANUAL entry point, and there it ran on a
# machine whose image carries no driver and simply refused to continue.
if ! command -v nvidia-smi >/dev/null 2>&1; then
if ! lspci 2>/dev/null | grep -qi nvidia; then
die "no NVIDIA device on the PCI bus - this is not a GPU instance."
fi
step "NVIDIA driver not present - installing (ubuntu-drivers --gpgpu)"
export DEBIAN_FRONTEND=noninteractive
$SUDO apt-get update -qq
$SUDO apt-get install -y -qq ubuntu-drivers-common pciutils >/dev/null
# β The fallback pin must satisfy the CUDA the TensorRT WHEELS link against,
# not merely boot the card. Our pin is a cu13 wheel (TensorRT 11.x), and
# `build_engines.sh` picks cu12 vs cu13 from the CUDA version the DRIVER
# reports. A 570 driver reports CUDA 12.x, so the pinned cu13 install would
# never be attempted: the script would ask for a cu12 wheel at our pin, fail,
# retry UNPINNED, and the version guard in build_trt_engine.py would then
# kill the run after the VM was already paid for. 595-server reports CUDA 13.2
# and is what `ubuntu-drivers --gpgpu` selects on Ubuntu 24.04 today
# (measured on an A5000, 2026-09-18).
$SUDO ubuntu-drivers install --gpgpu || $SUDO apt-get install -y -qq nvidia-driver-595-server
# ubuntu-drivers --gpgpu installs the kernel module and the compute libs but
# NOT nvidia-utils, which is the package that owns /usr/bin/nvidia-smi. The
# module then loads fine and the GPU works, while every arch-detection step
# here and in build_engines.sh keys off nvidia-smi and reads the machine as
# having no GPU. Measured 2026-08-28: lsmod showed nvidia/nvidia_uvm loaded
# and `nvidia-smi` was command-not-found.
BRANCH="$($SUDO dpkg -l 2>/dev/null | grep -o 'nvidia-compute-utils-[0-9]*-server' | head -1 | sed 's/nvidia-compute-utils-//')"
if [ -n "$BRANCH" ]; then
$SUDO apt-get install -y -qq "nvidia-utils-$BRANCH" >/dev/null 2>&1 || warn "could not install nvidia-utils-$BRANCH - nvidia-smi may be missing"
fi
if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1; then
ok "driver active without a reboot"
else
echo
warn "Driver installed, but the kernel module is not live until a REBOOT."
warn "Reboot, then run this script again - it will skip straight to the build:"
warn " reboot"
warn " curl -fsSL \"\$TOOLS_URL/bootstrap_linux.sh\" | bash"
exit 2
fi
fi
CC="$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d ' ')"
GPU="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | sed 's/^ *//')"
DRV="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1 | tr -d ' ')"
[ -n "$CC" ] || die "nvidia-smi reported no compute capability - driver present but GPU unusable."
SM="sm_${CC//./}"
ok "GPU: $GPU compute_cap $CC -> $SM driver $DRV"
. /etc/os-release 2>/dev/null || true
ok "OS: ${PRETTY_NAME:-unknown}"
# ββ 1. system packages ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# python3 + pip + curl only. TensorRT arrives as a pip wheel from
# build_trt_engine.py and CUDA comes from the driver - installing a CUDA toolkit
# would only add a second, possibly mismatched, libcuda to the search path.
step "System packages"
if command -v apt-get >/dev/null 2>&1; then
export DEBIAN_FRONTEND=noninteractive
$SUDO apt-get update -qq || warn "apt-get update failed - continuing with what is installed"
$SUDO apt-get install -y -qq python3 python3-pip python3-venv curl ca-certificates >/dev/null \
|| warn "apt-get install returned non-zero - continuing"
elif command -v dnf >/dev/null 2>&1; then
$SUDO dnf install -y -q python3 python3-pip curl ca-certificates >/dev/null || warn "dnf install returned non-zero"
elif command -v yum >/dev/null 2>&1; then
$SUDO yum install -y -q python3 python3-pip curl ca-certificates >/dev/null || warn "yum install returned non-zero"
else
warn "unknown package manager - assuming python3/pip/curl are already present"
fi
command -v "$PY" >/dev/null 2>&1 || die "$PY not found after package install."
ok "python: $($PY --version 2>&1)"
# PEP 668 marks distro pythons as externally managed; --break-system-packages is
# the sanctioned override and is exactly right on a throwaway build box.
PIPFLAGS=""
$PY -m pip install --dry-run --break-system-packages -q pip >/dev/null 2>&1 && PIPFLAGS="--break-system-packages"
$PY -m pip install $PIPFLAGS -q -U pip huggingface_hub >/dev/null 2>&1 \
|| warn "pip bootstrap returned non-zero - run_matrix will retry per-package"
ok "pip ready${PIPFLAGS:+ ($PIPFLAGS)}"
# ββ 2. fetch the build tools ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
step "Fetching build tools from $TOOLS_URL"
mkdir -p "$TOOLS"
cd "$TOOLS"
GOT=""; MISSING=""
for f in $TOOL_FILES; do
if curl -fsSL -o "$f" "${TOOLS_URL%/}/$f"; then
GOT="$GOT $f"
else
case " $TOOL_OPTIONAL " in
*" $f "*) MISSING="$MISSING $f" ;;
*) die "could not fetch $f from $TOOLS_URL - is the prefix public?" ;;
esac
fi
done
ok "fetched:$GOT"
[ -n "$MISSING" ] && warn "absent (fine unless an INT8 arch needs PTQ):$MISSING"
chmod +x build_engines.sh 2>/dev/null || true
# ββ 3. build ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
head_ "Linux lane - building linux_x64 engines on $SM"
ARGS=""
for m in ${ONLY:-}; do ARGS="$ARGS --only $m"; done
for m in ${SKIP:-}; do ARGS="$ARGS --skip $m"; done
TOOLS_URL="$TOOLS_URL" HF_TOKEN="$HF_TOKEN" DEST_PREFIX="$DEST_PREFIX" WORKDIR="$WORK" \
$PY run_matrix.py $ARGS
RC=$?
# ββ 4. report βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
head_ "RESULT - $GPU / $SM"
if [ "$RC" = "0" ]; then
echo -e " linux_x64 : ${G}OK${Z} (all models)"
else
echo -e " linux_x64 : ${R}$RC model(s) FAILED${Z}"
fi
echo
echo -e "${D} Verify what actually landed:${Z}"
echo -e "${D} https://huggingface.co/faxenoff/code-daemon-embed-v1/tree/main${Z}"
echo -e "${D} every name must end in _${SM}.engine and contain _linux_x64 for this box${Z}"
echo
echo -e "${D} The matching win_x64 cells come from bootstrap_win.ps1 on a Windows GPU VM${Z}"
echo -e "${D} with the same card - an engine built here will NOT load on Windows.${Z}"
exit $RC
|