code-daemon-embed-v1 / _build-tools /bootstrap_linux.sh
faxenoff's picture
cloud build tools (matrix trt11.3, 5 archs); the tools bucket is read-only since 10-07
40da26e verified
Raw History Blame Contribute Delete
10.2 kB
#!/usr/bin/env bash
# bootstrap_linux.sh - turn a bare Linux GPU VM into a linux_x64 TRT engine builder.
#
# The Linux twin of bootstrap_win.ps1, and the fallback when a Windows VM's WSL
# cannot see the GPU (Tesla cards in TCC mode, vGPU profiles, nested virt off -
# bootstrap_win diagnoses which). Everything it needs comes over HTTPS: the tools
# from TOOLS_URL, the ONNX from the public HF model repos, and the engines go back
# to those same repos. No S3 credentials touch the box.
#
# ── On-VM usage (the only command you type) ───────────────────────────────────────────
# export TOOLS_URL=https://<bucket>.s3.twcstorage.ru/_build-tools
# export HF_TOKEN=hf_...write...
# curl -fsSL "$TOOLS_URL/bootstrap_linux.sh" | bash
#
# Optional:
# DEST_PREFIX also copy the engines to S3 (needs S3_* creds too)
# DISTRO_PY python to use (default: python3)
# ONLY / SKIP space-separated model ids, passed through to run_matrix.py
set -uo pipefail
C='\033[0;36m'; G='\033[0;32m'; R='\033[0;31m'; Y='\033[0;33m'; D='\033[0;90m'; Z='\033[0m'
head_() { echo; echo -e "${D}$(printf '=%.0s' {1..72})${Z}"; echo -e "${C} $*${Z}"; echo -e "${D}$(printf '=%.0s' {1..72})${Z}"; }
step() { echo -e "\n${C}[>] $*${Z}"; }
ok() { echo -e " ${G}[OK]${Z} $*"; }
warn() { echo -e " ${Y}[!] ${Z} $*"; }
die() { echo -e " ${R}[!!]${Z} $*" >&2; exit 1; }
TOOLS_URL="${TOOLS_URL:-}"
HF_TOKEN="${HF_TOKEN:-}"
DEST_PREFIX="${DEST_PREFIX:-}"
PY="${DISTRO_PY:-python3}"
WORK="${WORKDIR:-$HOME/uc-build}"
TOOLS="$WORK/_tools"
# build_matrix.json is what makes the run self-describing: precision per model per
# arch comes from the catalog, not from whatever the operator remembered to export.
TOOL_FILES="run_matrix.py build_matrix.json build_engines.sh build_trt_engine.py onnx_fp16.py quantize_onnx.py calib_corpus.jsonl"
# fp16-only archs never touch the quantizer or the corpus - a 404 there must not
# block a Turing run. onnx_fp16.py is NOT optional: it is a local repo module the
# worker imports inside a try, so a missing one skips the FP16 pass silently and
# ships an FP32 engine under an FP16 name.
TOOL_OPTIONAL="quantize_onnx.py calib_corpus.jsonl"
head_ "ultracode cloud builder - Linux VM bootstrap"
[ -n "$TOOLS_URL" ] || die "set TOOLS_URL first, e.g. export TOOLS_URL=https://<bucket>.s3.twcstorage.ru/_build-tools"
if [ -z "$HF_TOKEN" ] && [ -z "$DEST_PREFIX" ]; then
die "set HF_TOKEN (a WRITE token) or DEST_PREFIX - otherwise the engines are built and discarded."
fi
# ── 0. preflight ──────────────────────────────────────────────────────────────────────
step "Preflight"
[ "$(id -u)" = "0" ] && SUDO="" || SUDO="sudo"
if [ -n "$SUDO" ] && ! command -v sudo >/dev/null 2>&1; then
die "not root and no sudo - run as root or install sudo."
fi
# Install the driver rather than demanding one. The Terraform lane has
# cloud-init's engine-build.sh do this before calling us, so the ordering never
# showed - but this script is also the MANUAL entry point, and there it ran on a
# machine whose image carries no driver and simply refused to continue.
if ! command -v nvidia-smi >/dev/null 2>&1; then
if ! lspci 2>/dev/null | grep -qi nvidia; then
die "no NVIDIA device on the PCI bus - this is not a GPU instance."
fi
step "NVIDIA driver not present - installing (ubuntu-drivers --gpgpu)"
export DEBIAN_FRONTEND=noninteractive
$SUDO apt-get update -qq
$SUDO apt-get install -y -qq ubuntu-drivers-common pciutils >/dev/null
# β›” The fallback pin must satisfy the CUDA the TensorRT WHEELS link against,
# not merely boot the card. Our pin is a cu13 wheel (TensorRT 11.x), and
# `build_engines.sh` picks cu12 vs cu13 from the CUDA version the DRIVER
# reports. A 570 driver reports CUDA 12.x, so the pinned cu13 install would
# never be attempted: the script would ask for a cu12 wheel at our pin, fail,
# retry UNPINNED, and the version guard in build_trt_engine.py would then
# kill the run after the VM was already paid for. 595-server reports CUDA 13.2
# and is what `ubuntu-drivers --gpgpu` selects on Ubuntu 24.04 today
# (measured on an A5000, 2026-09-18).
$SUDO ubuntu-drivers install --gpgpu || $SUDO apt-get install -y -qq nvidia-driver-595-server
# ubuntu-drivers --gpgpu installs the kernel module and the compute libs but
# NOT nvidia-utils, which is the package that owns /usr/bin/nvidia-smi. The
# module then loads fine and the GPU works, while every arch-detection step
# here and in build_engines.sh keys off nvidia-smi and reads the machine as
# having no GPU. Measured 2026-08-28: lsmod showed nvidia/nvidia_uvm loaded
# and `nvidia-smi` was command-not-found.
BRANCH="$($SUDO dpkg -l 2>/dev/null | grep -o 'nvidia-compute-utils-[0-9]*-server' | head -1 | sed 's/nvidia-compute-utils-//')"
if [ -n "$BRANCH" ]; then
$SUDO apt-get install -y -qq "nvidia-utils-$BRANCH" >/dev/null 2>&1 || warn "could not install nvidia-utils-$BRANCH - nvidia-smi may be missing"
fi
if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1; then
ok "driver active without a reboot"
else
echo
warn "Driver installed, but the kernel module is not live until a REBOOT."
warn "Reboot, then run this script again - it will skip straight to the build:"
warn " reboot"
warn " curl -fsSL \"\$TOOLS_URL/bootstrap_linux.sh\" | bash"
exit 2
fi
fi
CC="$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d ' ')"
GPU="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | sed 's/^ *//')"
DRV="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1 | tr -d ' ')"
[ -n "$CC" ] || die "nvidia-smi reported no compute capability - driver present but GPU unusable."
SM="sm_${CC//./}"
ok "GPU: $GPU compute_cap $CC -> $SM driver $DRV"
. /etc/os-release 2>/dev/null || true
ok "OS: ${PRETTY_NAME:-unknown}"
# ── 1. system packages ────────────────────────────────────────────────────────────────
# python3 + pip + curl only. TensorRT arrives as a pip wheel from
# build_trt_engine.py and CUDA comes from the driver - installing a CUDA toolkit
# would only add a second, possibly mismatched, libcuda to the search path.
step "System packages"
if command -v apt-get >/dev/null 2>&1; then
export DEBIAN_FRONTEND=noninteractive
$SUDO apt-get update -qq || warn "apt-get update failed - continuing with what is installed"
$SUDO apt-get install -y -qq python3 python3-pip python3-venv curl ca-certificates >/dev/null \
|| warn "apt-get install returned non-zero - continuing"
elif command -v dnf >/dev/null 2>&1; then
$SUDO dnf install -y -q python3 python3-pip curl ca-certificates >/dev/null || warn "dnf install returned non-zero"
elif command -v yum >/dev/null 2>&1; then
$SUDO yum install -y -q python3 python3-pip curl ca-certificates >/dev/null || warn "yum install returned non-zero"
else
warn "unknown package manager - assuming python3/pip/curl are already present"
fi
command -v "$PY" >/dev/null 2>&1 || die "$PY not found after package install."
ok "python: $($PY --version 2>&1)"
# PEP 668 marks distro pythons as externally managed; --break-system-packages is
# the sanctioned override and is exactly right on a throwaway build box.
PIPFLAGS=""
$PY -m pip install --dry-run --break-system-packages -q pip >/dev/null 2>&1 && PIPFLAGS="--break-system-packages"
$PY -m pip install $PIPFLAGS -q -U pip huggingface_hub >/dev/null 2>&1 \
|| warn "pip bootstrap returned non-zero - run_matrix will retry per-package"
ok "pip ready${PIPFLAGS:+ ($PIPFLAGS)}"
# ── 2. fetch the build tools ──────────────────────────────────────────────────────────
step "Fetching build tools from $TOOLS_URL"
mkdir -p "$TOOLS"
cd "$TOOLS"
GOT=""; MISSING=""
for f in $TOOL_FILES; do
if curl -fsSL -o "$f" "${TOOLS_URL%/}/$f"; then
GOT="$GOT $f"
else
case " $TOOL_OPTIONAL " in
*" $f "*) MISSING="$MISSING $f" ;;
*) die "could not fetch $f from $TOOLS_URL - is the prefix public?" ;;
esac
fi
done
ok "fetched:$GOT"
[ -n "$MISSING" ] && warn "absent (fine unless an INT8 arch needs PTQ):$MISSING"
chmod +x build_engines.sh 2>/dev/null || true
# ── 3. build ──────────────────────────────────────────────────────────────────────────
head_ "Linux lane - building linux_x64 engines on $SM"
ARGS=""
for m in ${ONLY:-}; do ARGS="$ARGS --only $m"; done
for m in ${SKIP:-}; do ARGS="$ARGS --skip $m"; done
TOOLS_URL="$TOOLS_URL" HF_TOKEN="$HF_TOKEN" DEST_PREFIX="$DEST_PREFIX" WORKDIR="$WORK" \
$PY run_matrix.py $ARGS
RC=$?
# ── 4. report ─────────────────────────────────────────────────────────────────────────
head_ "RESULT - $GPU / $SM"
if [ "$RC" = "0" ]; then
echo -e " linux_x64 : ${G}OK${Z} (all models)"
else
echo -e " linux_x64 : ${R}$RC model(s) FAILED${Z}"
fi
echo
echo -e "${D} Verify what actually landed:${Z}"
echo -e "${D} https://huggingface.co/faxenoff/code-daemon-embed-v1/tree/main${Z}"
echo -e "${D} every name must end in _${SM}.engine and contain _linux_x64 for this box${Z}"
echo
echo -e "${D} The matching win_x64 cells come from bootstrap_win.ps1 on a Windows GPU VM${Z}"
echo -e "${D} with the same card - an engine built here will NOT load on Windows.${Z}"
exit $RC