#!/usr/bin/env bash # bootstrap_linux.sh - turn a bare Linux GPU VM into a linux_x64 TRT engine builder. # # The Linux twin of bootstrap_win.ps1, and the fallback when a Windows VM's WSL # cannot see the GPU (Tesla cards in TCC mode, vGPU profiles, nested virt off - # bootstrap_win diagnoses which). Everything it needs comes over HTTPS: the tools # from TOOLS_URL, the ONNX from the public HF model repos, and the engines go back # to those same repos. No S3 credentials touch the box. # # ── On-VM usage (the only command you type) ─────────────────────────────────────────── # export TOOLS_URL=https://.s3.twcstorage.ru/_build-tools # export HF_TOKEN=hf_...write... # curl -fsSL "$TOOLS_URL/bootstrap_linux.sh" | bash # # Optional: # DEST_PREFIX also copy the engines to S3 (needs S3_* creds too) # DISTRO_PY python to use (default: python3) # ONLY / SKIP space-separated model ids, passed through to run_matrix.py set -uo pipefail C='\033[0;36m'; G='\033[0;32m'; R='\033[0;31m'; Y='\033[0;33m'; D='\033[0;90m'; Z='\033[0m' head_() { echo; echo -e "${D}$(printf '=%.0s' {1..72})${Z}"; echo -e "${C} $*${Z}"; echo -e "${D}$(printf '=%.0s' {1..72})${Z}"; } step() { echo -e "\n${C}[>] $*${Z}"; } ok() { echo -e " ${G}[OK]${Z} $*"; } warn() { echo -e " ${Y}[!] ${Z} $*"; } die() { echo -e " ${R}[!!]${Z} $*" >&2; exit 1; } TOOLS_URL="${TOOLS_URL:-}" HF_TOKEN="${HF_TOKEN:-}" DEST_PREFIX="${DEST_PREFIX:-}" PY="${DISTRO_PY:-python3}" WORK="${WORKDIR:-$HOME/uc-build}" TOOLS="$WORK/_tools" # build_matrix.json is what makes the run self-describing: precision per model per # arch comes from the catalog, not from whatever the operator remembered to export. TOOL_FILES="run_matrix.py build_matrix.json build_engines.sh build_trt_engine.py onnx_fp16.py quantize_onnx.py calib_corpus.jsonl" # fp16-only archs never touch the quantizer or the corpus - a 404 there must not # block a Turing run. onnx_fp16.py is NOT optional: it is a local repo module the # worker imports inside a try, so a missing one skips the FP16 pass silently and # ships an FP32 engine under an FP16 name. TOOL_OPTIONAL="quantize_onnx.py calib_corpus.jsonl" head_ "ultracode cloud builder - Linux VM bootstrap" [ -n "$TOOLS_URL" ] || die "set TOOLS_URL first, e.g. export TOOLS_URL=https://.s3.twcstorage.ru/_build-tools" if [ -z "$HF_TOKEN" ] && [ -z "$DEST_PREFIX" ]; then die "set HF_TOKEN (a WRITE token) or DEST_PREFIX - otherwise the engines are built and discarded." fi # ── 0. preflight ────────────────────────────────────────────────────────────────────── step "Preflight" [ "$(id -u)" = "0" ] && SUDO="" || SUDO="sudo" if [ -n "$SUDO" ] && ! command -v sudo >/dev/null 2>&1; then die "not root and no sudo - run as root or install sudo." fi # Install the driver rather than demanding one. The Terraform lane has # cloud-init's engine-build.sh do this before calling us, so the ordering never # showed - but this script is also the MANUAL entry point, and there it ran on a # machine whose image carries no driver and simply refused to continue. if ! command -v nvidia-smi >/dev/null 2>&1; then if ! lspci 2>/dev/null | grep -qi nvidia; then die "no NVIDIA device on the PCI bus - this is not a GPU instance." fi step "NVIDIA driver not present - installing (ubuntu-drivers --gpgpu)" export DEBIAN_FRONTEND=noninteractive $SUDO apt-get update -qq $SUDO apt-get install -y -qq ubuntu-drivers-common pciutils >/dev/null # ⛔ The fallback pin must satisfy the CUDA the TensorRT WHEELS link against, # not merely boot the card. Our pin is a cu13 wheel (TensorRT 11.x), and # `build_engines.sh` picks cu12 vs cu13 from the CUDA version the DRIVER # reports. A 570 driver reports CUDA 12.x, so the pinned cu13 install would # never be attempted: the script would ask for a cu12 wheel at our pin, fail, # retry UNPINNED, and the version guard in build_trt_engine.py would then # kill the run after the VM was already paid for. 595-server reports CUDA 13.2 # and is what `ubuntu-drivers --gpgpu` selects on Ubuntu 24.04 today # (measured on an A5000, 2026-09-18). $SUDO ubuntu-drivers install --gpgpu || $SUDO apt-get install -y -qq nvidia-driver-595-server # ubuntu-drivers --gpgpu installs the kernel module and the compute libs but # NOT nvidia-utils, which is the package that owns /usr/bin/nvidia-smi. The # module then loads fine and the GPU works, while every arch-detection step # here and in build_engines.sh keys off nvidia-smi and reads the machine as # having no GPU. Measured 2026-08-28: lsmod showed nvidia/nvidia_uvm loaded # and `nvidia-smi` was command-not-found. BRANCH="$($SUDO dpkg -l 2>/dev/null | grep -o 'nvidia-compute-utils-[0-9]*-server' | head -1 | sed 's/nvidia-compute-utils-//')" if [ -n "$BRANCH" ]; then $SUDO apt-get install -y -qq "nvidia-utils-$BRANCH" >/dev/null 2>&1 || warn "could not install nvidia-utils-$BRANCH - nvidia-smi may be missing" fi if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1; then ok "driver active without a reboot" else echo warn "Driver installed, but the kernel module is not live until a REBOOT." warn "Reboot, then run this script again - it will skip straight to the build:" warn " reboot" warn " curl -fsSL \"\$TOOLS_URL/bootstrap_linux.sh\" | bash" exit 2 fi fi CC="$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d ' ')" GPU="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | sed 's/^ *//')" DRV="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1 | tr -d ' ')" [ -n "$CC" ] || die "nvidia-smi reported no compute capability - driver present but GPU unusable." SM="sm_${CC//./}" ok "GPU: $GPU compute_cap $CC -> $SM driver $DRV" . /etc/os-release 2>/dev/null || true ok "OS: ${PRETTY_NAME:-unknown}" # ── 1. system packages ──────────────────────────────────────────────────────────────── # python3 + pip + curl only. TensorRT arrives as a pip wheel from # build_trt_engine.py and CUDA comes from the driver - installing a CUDA toolkit # would only add a second, possibly mismatched, libcuda to the search path. step "System packages" if command -v apt-get >/dev/null 2>&1; then export DEBIAN_FRONTEND=noninteractive $SUDO apt-get update -qq || warn "apt-get update failed - continuing with what is installed" $SUDO apt-get install -y -qq python3 python3-pip python3-venv curl ca-certificates >/dev/null \ || warn "apt-get install returned non-zero - continuing" elif command -v dnf >/dev/null 2>&1; then $SUDO dnf install -y -q python3 python3-pip curl ca-certificates >/dev/null || warn "dnf install returned non-zero" elif command -v yum >/dev/null 2>&1; then $SUDO yum install -y -q python3 python3-pip curl ca-certificates >/dev/null || warn "yum install returned non-zero" else warn "unknown package manager - assuming python3/pip/curl are already present" fi command -v "$PY" >/dev/null 2>&1 || die "$PY not found after package install." ok "python: $($PY --version 2>&1)" # PEP 668 marks distro pythons as externally managed; --break-system-packages is # the sanctioned override and is exactly right on a throwaway build box. PIPFLAGS="" $PY -m pip install --dry-run --break-system-packages -q pip >/dev/null 2>&1 && PIPFLAGS="--break-system-packages" $PY -m pip install $PIPFLAGS -q -U pip huggingface_hub >/dev/null 2>&1 \ || warn "pip bootstrap returned non-zero - run_matrix will retry per-package" ok "pip ready${PIPFLAGS:+ ($PIPFLAGS)}" # ── 2. fetch the build tools ────────────────────────────────────────────────────────── step "Fetching build tools from $TOOLS_URL" mkdir -p "$TOOLS" cd "$TOOLS" GOT=""; MISSING="" for f in $TOOL_FILES; do if curl -fsSL -o "$f" "${TOOLS_URL%/}/$f"; then GOT="$GOT $f" else case " $TOOL_OPTIONAL " in *" $f "*) MISSING="$MISSING $f" ;; *) die "could not fetch $f from $TOOLS_URL - is the prefix public?" ;; esac fi done ok "fetched:$GOT" [ -n "$MISSING" ] && warn "absent (fine unless an INT8 arch needs PTQ):$MISSING" chmod +x build_engines.sh 2>/dev/null || true # ── 3. build ────────────────────────────────────────────────────────────────────────── head_ "Linux lane - building linux_x64 engines on $SM" ARGS="" for m in ${ONLY:-}; do ARGS="$ARGS --only $m"; done for m in ${SKIP:-}; do ARGS="$ARGS --skip $m"; done TOOLS_URL="$TOOLS_URL" HF_TOKEN="$HF_TOKEN" DEST_PREFIX="$DEST_PREFIX" WORKDIR="$WORK" \ $PY run_matrix.py $ARGS RC=$? # ── 4. report ───────────────────────────────────────────────────────────────────────── head_ "RESULT - $GPU / $SM" if [ "$RC" = "0" ]; then echo -e " linux_x64 : ${G}OK${Z} (all models)" else echo -e " linux_x64 : ${R}$RC model(s) FAILED${Z}" fi echo echo -e "${D} Verify what actually landed:${Z}" echo -e "${D} https://huggingface.co/faxenoff/code-daemon-embed-v1/tree/main${Z}" echo -e "${D} every name must end in _${SM}.engine and contain _linux_x64 for this box${Z}" echo echo -e "${D} The matching win_x64 cells come from bootstrap_win.ps1 on a Windows GPU VM${Z}" echo -e "${D} with the same card - an engine built here will NOT load on Windows.${Z}" exit $RC