code-daemon-embed-v1 / _build-tools /bootstrap_win.ps1
faxenoff's picture
cloud build tools (matrix trt11.3, 5 archs); the tools bucket is read-only since 10-07
40da26e verified
Raw History Blame Contribute Delete
22.4 kB
# bootstrap_win.ps1 - turn a bare Windows GPU VM into a two-OS TRT engine builder.
#
# One rented Windows box can produce BOTH halves of a {win_x64, linux_x64} pair:
# the Windows engines natively, the Linux engines inside WSL2 against the same
# physical GPU. That halves the rental - and, more importantly, it guarantees the
# two halves saw the identical GPU, driver and ONNX, so a difference between them
# is a real difference and not a difference between two rented machines.
#
# The WSL half is BEST EFFORT and says so loudly. GPU passthrough into WSL needs a
# WDDM-mode driver and nested virtualization on the hypervisor; a Tesla card left
# in TCC mode or a VM without nested virt will not have it, and this script
# diagnoses which of those it is rather than reporting a mysterious skip.
#
# -- On-VM usage (the only command you type) -------------------------------------------
# $env:TOOLS_URL = "https://<bucket>.s3.twcstorage.ru/_build-tools"
# $env:HF_TOKEN = "hf_...write..."
# irm "$($env:TOOLS_URL)/bootstrap_win.ps1" | iex
#
# Everything else is discovered: GPU arch, which models to build, at what precision.
#
# To pass a switch through the pipe, `iex` is not enough - it takes no arguments:
# & ([scriptblock]::Create((irm "$($env:TOOLS_URL)/bootstrap_win.ps1"))) -Stage wsl
#
# -Stage win Windows lane only (skip WSL entirely)
# -Stage wsl WSL lane only (assumes the Windows lane already ran)
# -Stage all both (default)
# -NoReboot never schedule a reboot; report what WSL still needs instead
# -Distro WSL distribution (default Ubuntu-24.04)
[CmdletBinding()]
param(
[ValidateSet("all", "win", "wsl")] [string] $Stage = "all",
[switch] $NoReboot,
[string] $Distro = "Ubuntu-24.04"
)
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue" # Invoke-WebRequest is ~10x faster without it
# -- output helpers --------------------------------------------------------------------
function Head($m) { Write-Host ""; Write-Host ("=" * 72) -ForegroundColor DarkGray; Write-Host " $m" -ForegroundColor Cyan; Write-Host ("=" * 72) -ForegroundColor DarkGray }
function Step($m) { Write-Host "`n[>] $m" -ForegroundColor Cyan }
function Ok($m) { Write-Host " [OK] $m" -ForegroundColor Green }
function Warn($m) { Write-Host " [!] $m" -ForegroundColor Yellow }
function Die($m) { Write-Host " [!!] $m" -ForegroundColor Red; exit 1 }
$TOOLS_URL = [Environment]::GetEnvironmentVariable("TOOLS_URL")
$HF_TOKEN = [Environment]::GetEnvironmentVariable("HF_TOKEN")
$DEST_PREFIX = [Environment]::GetEnvironmentVariable("DEST_PREFIX")
$WORK = Join-Path $env:USERPROFILE "uc-build"
$TOOLS = Join-Path $WORK "_tools"
$STATE = Join-Path $WORK "bootstrap-state.json"
# The files a build needs on the box. build_matrix.json is what makes the run
# self-describing: precision per model per arch comes from the catalog, not from
# whatever the operator remembered to export.
$TOOL_FILES = @(
"run_matrix.py", "build_matrix.json",
"build_engines.ps1", "build_engines.sh",
"build_trt_engine.py", "quantize_onnx.py", "onnx_fp16.py", "calib_corpus.jsonl"
)
# Missing one of these is not fatal: fp16-only archs never touch the quantizer or
# the corpus, and paying for a 404 there would block a Turing run for no reason.
$TOOL_OPTIONAL = @("quantize_onnx.py", "calib_corpus.jsonl")
Head "ultracode cloud builder - Windows VM bootstrap"
if (-not $TOOLS_URL) { Die "set TOOLS_URL first, e.g. `$env:TOOLS_URL='https://<bucket>.s3.twcstorage.ru/_build-tools'" }
if (-not $HF_TOKEN -and -not $DEST_PREFIX) {
Die "set HF_TOKEN (a WRITE token) or DEST_PREFIX - otherwise the engines are built and discarded."
}
# -- 0. preflight: is this machine even eligible? --------------------------------------
Step "Preflight"
$isAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()
).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
if (-not $isAdmin) { Die "run this in an ELEVATED PowerShell - installing Python and WSL needs it." }
Ok "elevated"
$osv = [Environment]::OSVersion.Version
$caption = (Get-CimInstance Win32_OperatingSystem).Caption
Ok ("{0} (build {1})" -f $caption, $osv.Build)
if ($osv.Build -lt 19041) {
# Server 2019 is 17763 and is what Selectel offers, so this is the normal
# case there rather than a problem: win_x64 builds here, linux_x64 comes from
# a separate Ubuntu VM. Stated once, plainly, so it does not read as a fault.
Write-Host " [i] build < 19041 - no WSL2 on this image. win_x64 builds here;" -ForegroundColor DarkGray
Write-Host " linux_x64 comes from a Linux VM (bootstrap_linux.sh). Expected on Server 2019." -ForegroundColor DarkGray
}
if (-not (Get-Command nvidia-smi -ErrorAction SilentlyContinue)) {
Die "nvidia-smi not on PATH - install the NVIDIA driver first. On a fresh cloud VM the driver is often NOT preinstalled; ask the provider for a GPU image or install from nvidia.com."
}
$cc = (& nvidia-smi --query-gpu=compute_cap --format=csv,noheader | Select-Object -First 1).Trim()
$gpu = (& nvidia-smi --query-gpu=name --format=csv,noheader | Select-Object -First 1).Trim()
$drv = (& nvidia-smi --query-gpu=driver_version --format=csv,noheader | Select-Object -First 1).Trim()
if (-not $cc) { Die "nvidia-smi reported no compute capability - the driver is present but the GPU is not usable." }
$SM = "sm_" + ($cc -replace '\.', '')
Ok ("GPU: {0} compute_cap {1} -> {2} driver {3}" -f $gpu, $cc, $SM, $drv)
# The driver model decides whether WSL can see this GPU at all. TCC (Tesla Compute
# Cluster) has no WDDM display stack, and /dev/dxg - the whole basis of WSL GPU
# passthrough - is a WDDM construct. Read it now so the WSL stage can explain
# itself instead of just failing.
$driverModel = ""
try {
$dm = (& nvidia-smi --query-gpu=driver_model.current --format=csv,noheader 2>$null | Select-Object -First 1)
if ($dm) { $driverModel = $dm.Trim() }
} catch { }
if ($driverModel) { Ok ("driver model: {0}" -f $driverModel) }
# -- 1. Python 3.13 - the cu13 TensorRT wheels are built for it ------------------------
Step "Python"
function Find-Py313 {
# Return the FULL PATH, never the bare name "py".
#
# PowerShell resolves a command as alias -> FUNCTION -> cmdlet -> program, and
# is case-insensitive. With `$PyExe = "py"` and a helper function named `Py`
# in scope, `& $PyExe` calls the FUNCTION, which calls `& $PyExe` again:
# infinite recursion, ending in "The script failed due to call depth
# overflow" right after the Python step announces itself. A path can never
# resolve to a function.
$launcher = Get-Command py.exe -CommandType Application -ErrorAction SilentlyContinue |
Select-Object -First 1
if ($launcher) {
& $launcher.Source -3.13 -c "import sys" *> $null
if ($LASTEXITCODE -eq 0) { return @{ Exe = $launcher.Source; Pre = @("-3.13") } }
}
$direct = Join-Path $env:LOCALAPPDATA "Programs\Python\Python313\python.exe"
if (Test-Path $direct) { return @{ Exe = $direct; Pre = @() } }
$direct = Join-Path $env:ProgramFiles "Python313\python.exe"
if (Test-Path $direct) { return @{ Exe = $direct; Pre = @() } }
return $null
}
$py = Find-Py313
if (-not $py) {
Warn "Python 3.13 not found - installing (the cu13 tensorrt wheels have no 3.12/3.11 build)"
$installer = Join-Path $env:TEMP "python-3.13-amd64.exe"
# python.org direct, not winget: winget is absent on Server images and its
# Store-backed Python is a stub that answers `python` with a store page.
$url = "https://www.python.org/ftp/python/3.13.7/python-3.13.7-amd64.exe"
Invoke-WebRequest -Uri $url -OutFile $installer -UseBasicParsing
# PrependPath so the WSL side and any later shell find it too; Include_launcher
# gives us `py -3.13`, which is what build_engines.ps1 looks for.
$p = Start-Process -FilePath $installer -Wait -PassThru -ArgumentList @(
"/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_launcher=1", "Include_pip=1"
)
if ($p.ExitCode -ne 0) { Die ("python installer exited {0}" -f $p.ExitCode) }
$env:Path = [Environment]::GetEnvironmentVariable("Path", "Machine") + ";" +
[Environment]::GetEnvironmentVariable("Path", "User")
$py = Find-Py313
if (-not $py) { Die "Python 3.13 still not found after install." }
}
$PyExe = $py.Exe; $PyPre = $py.Pre
function Invoke-Py { & $PyExe @PyPre @args; if ($LASTEXITCODE -ne 0) { throw ("python exited {0}" -f $LASTEXITCODE) } }
Ok ("python: {0}" -f ((& $PyExe @PyPre --version) 2>&1))
# -- 2. fetch the build tools ----------------------------------------------------------
Step "Fetching build tools from $TOOLS_URL"
New-Item -ItemType Directory -Force -Path $WORK, $TOOLS | Out-Null
$fetched = @(); $missing = @()
foreach ($f in $TOOL_FILES) {
$dst = Join-Path $TOOLS $f
try {
Invoke-WebRequest -Uri ("{0}/{1}" -f $TOOLS_URL.TrimEnd('/'), $f) -OutFile $dst -UseBasicParsing
$fetched += $f
} catch {
if ($TOOL_OPTIONAL -contains $f) { $missing += $f } else { Die ("could not fetch {0} from {1} - is the prefix public? ({2})" -f $f, $TOOLS_URL, $_.Exception.Message) }
}
}
Ok ("fetched: {0}" -f ($fetched -join ", "))
if ($missing) { Warn ("absent (fine unless an INT8 arch needs PTQ): {0}" -f ($missing -join ", ")) }
# -- 3. the Windows lane ---------------------------------------------------------------
$winRc = $null
if ($Stage -eq "all" -or $Stage -eq "win") {
Head "Windows lane - building win_x64 engines on $SM"
$env:TOOLS_URL = $TOOLS_URL
$env:WORKDIR = $WORK
Push-Location $TOOLS
try {
& $PyExe @PyPre "run_matrix.py"
$winRc = $LASTEXITCODE
} finally { Pop-Location }
if ($winRc -eq 0) { Ok "win_x64 lane complete - every model uploaded" }
else { Warn ("win_x64 lane: {0} model(s) FAILED - see the log above" -f $winRc) }
}
# -- 4. the WSL lane -------------------------------------------------------------------
function Test-WslFeatures {
<# Both features must be Enabled before a distro can run in WSL2 mode.
Reported separately because the failure modes differ: no VirtualMachinePlatform
usually means nested virtualization is off on the hypervisor, which is a
provider-side setting no amount of local work fixes. #>
$res = @{}
foreach ($n in @("Microsoft-Windows-Subsystem-Linux", "VirtualMachinePlatform")) {
try { $res[$n] = (Get-WindowsOptionalFeature -Online -FeatureName $n).State }
catch { $res[$n] = "Unknown" }
}
return $res
}
function Enable-WslFeatures {
$needReboot = $false
foreach ($n in @("Microsoft-Windows-Subsystem-Linux", "VirtualMachinePlatform")) {
$st = (Get-WindowsOptionalFeature -Online -FeatureName $n).State
if ($st -ne "Enabled") {
Step "Enabling Windows feature $n"
$r = Enable-WindowsOptionalFeature -Online -FeatureName $n -All -NoRestart
if ($r.RestartNeeded) { $needReboot = $true }
}
}
return $needReboot
}
function Register-Resume {
<# A RunOnce entry, not a scheduled task: RunOnce fires on the next
interactive logon of THIS user with the console attached, which is what an
operator reconnecting over RDP actually gets. A SYSTEM scheduled task
would run headless and its output would go nowhere.
The token does NOT go in the registry. A RunOnce value is world-readable
and survives in registry backups and any image the provider snapshots; a
file with an Administrators-only ACL, deleted by the resume script as its
first act, keeps the blast radius to the reboot window. #>
$tokFile = Join-Path $WORK "resume-token.txt"
Set-Content -Path $tokFile -Value $HF_TOKEN -Encoding ascii -NoNewline
$acl = Get-Acl $tokFile
$acl.SetAccessRuleProtection($true, $false) # drop inherited user-readable ACEs
$acl.SetAccessRule((New-Object System.Security.AccessControl.FileSystemAccessRule(
"BUILTIN\Administrators", "FullControl", "Allow")))
Set-Acl -Path $tokFile -AclObject $acl
# `iex` takes no -Args; a scriptblock does. This is the only shape that can
# pass -Stage through a piped one-liner.
$inner = ("`$env:TOOLS_URL='{0}'; " +
"`$env:HF_TOKEN=(Get-Content '{1}' -Raw); Remove-Item '{1}' -Force; " +
"& ([scriptblock]::Create((irm '{0}/bootstrap_win.ps1'))) -Stage wsl") -f $TOOLS_URL, $tokFile
$enc = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($inner))
$cmd = "powershell -NoProfile -ExecutionPolicy Bypass -EncodedCommand $enc"
New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\RunOnce" `
-Name "UltracodeBuildResume" -Value $cmd -PropertyType String -Force | Out-Null
Warn "the WSL lane will resume automatically at the next logon (RunOnce)"
Warn ("token parked at {0} (Administrators only); the resume deletes it before building" -f $tokFile)
}
$wslRc = $null
$wslSkipReason = $null
if ($Stage -eq "all" -or $Stage -eq "wsl") {
Head "WSL lane - building linux_x64 engines on the same $SM GPU"
if ($osv.Build -lt 19041) {
$wslSkipReason = "Windows build $($osv.Build) < 19041 - WSL2 unavailable on this image"
} else {
$feat = Test-WslFeatures
foreach ($k in $feat.Keys) { Write-Host (" feature {0,-38} {1}" -f $k, $feat[$k]) }
if ($feat["VirtualMachinePlatform"] -ne "Enabled" -or
$feat["Microsoft-Windows-Subsystem-Linux"] -ne "Enabled") {
if ($NoReboot) {
$wslSkipReason = "WSL features are off and -NoReboot was given. Enable them and rerun with -Stage wsl."
} else {
$need = Enable-WslFeatures
if ($need) {
Register-Resume
Head "REBOOT REQUIRED"
Write-Host " Windows features were just enabled. The machine reboots in 15 seconds;"
Write-Host " log back in and the WSL lane resumes on its own."
if ($winRc -ne $null -and $winRc -ne 0) {
Warn ("the Windows lane had {0} failure(s) - re-run -Stage win after the reboot" -f $winRc)
}
Start-Sleep -Seconds 15
Restart-Computer -Force
exit 0
}
}
}
}
if (-not $wslSkipReason) {
# Is a distro registered? `wsl -l -q` output is UTF-16 on older builds -
# match loosely rather than parsing it as a clean list.
$distros = (& wsl.exe -l -q 2>$null) -join "`n"
$haveDistro = $distros -match [regex]::Escape($Distro.Split('-')[0])
if (-not $haveDistro) {
Step "Installing WSL distribution $Distro"
# --no-launch registers without dropping into the interactive
# "create a user" prompt, which would hang an unattended run forever.
# --web-download avoids the Microsoft Store, absent on Server images.
& wsl.exe --install --no-launch --web-download -d $Distro
if ($LASTEXITCODE -ne 0) {
Warn "wsl --install -d $Distro failed; retrying without --web-download"
& wsl.exe --install --no-launch -d $Distro
}
if ($LASTEXITCODE -ne 0) { $wslSkipReason = "could not install the $Distro distribution (wsl --install exited $LASTEXITCODE)" }
else {
# First run finishes registration. -u root skips user creation.
& wsl.exe -d $Distro -u root -- true 2>$null | Out-Null
}
}
}
if (-not $wslSkipReason) {
Step "Probing the GPU inside WSL"
$wslSmi = (& wsl.exe -d $Distro -u root -- bash -lc "nvidia-smi --query-gpu=name,compute_cap --format=csv,noheader 2>&1") -join " "
if ($LASTEXITCODE -ne 0 -or $wslSmi -notmatch '\d\.\d') {
# This is the failure worth diagnosing precisely - it is the single
# most likely reason the Linux half of the matrix does not get built.
Warn "nvidia-smi inside WSL did not report a GPU."
Write-Host " raw: $wslSmi" -ForegroundColor DarkGray
Write-Host ""
Write-Host " Why this happens, in order of likelihood on a cloud VM:" -ForegroundColor Yellow
if ($driverModel -match "TCC") {
Write-Host " 1. THIS ONE: the driver is in TCC mode ($driverModel). WSL needs WDDM." -ForegroundColor Red
Write-Host " Try: nvidia-smi -dm 0 then reboot, then rerun with -Stage wsl"
Write-Host " (Datacenter cards default to TCC; some vGPU profiles cannot switch.)"
} else {
Write-Host " 1. Driver model must be WDDM, not TCC. Check: nvidia-smi -q | findstr /i `"Driver Model`""
Write-Host " Switch with: nvidia-smi -dm 0 (then reboot)"
}
Write-Host " 2. NVIDIA vGPU / GRID profiles do not expose /dev/dxg to WSL at all."
Write-Host " If this is a vGPU VM, the Linux engines need a separate Linux VM."
Write-Host " 3. Nested virtualization off on the hypervisor. In VMware/Cloud Director:"
Write-Host " the VM needs 'Expose hardware assisted virtualization' (nested_hypervisor_enabled)."
Write-Host " 4. Driver older than 465 - no WSL CUDA support at all. This one is $drv."
Write-Host ""
$wslSkipReason = "no GPU visible inside WSL (see the diagnosis above)"
} else {
Ok ("WSL sees: {0}" -f $wslSmi.Trim())
}
}
if (-not $wslSkipReason) {
Step "Preparing the WSL toolchain"
# python3-pip + venv only. TensorRT arrives as a pip wheel from
# build_trt_engine.py, and CUDA itself comes from the Windows driver via
# /usr/lib/wsl/lib - installing a CUDA toolkit in the distro would only
# shadow it with a second, wrong libcuda.
$prep = @(
"export DEBIAN_FRONTEND=noninteractive",
"apt-get update -qq",
"apt-get install -y -qq python3 python3-pip python3-venv curl ca-certificates >/dev/null",
"python3 -m pip install --break-system-packages -q -U pip huggingface_hub 2>/dev/null || python3 -m pip install -q -U pip huggingface_hub"
) -join " && "
& wsl.exe -d $Distro -u root -- bash -lc "$prep"
if ($LASTEXITCODE -ne 0) { Warn "apt/pip prep returned $LASTEXITCODE - continuing, the build may still work" }
Step "Running the Linux matrix inside WSL"
# The tools are re-fetched INSIDE the distro rather than read across
# /mnt/c: DrvFs has no executable bit and its inode semantics confuse
# anything that stats a file twice. A second download costs seconds.
# Single-quoted: `$HOME` is expanded by bash inside the distro (root's home
# there), never by PowerShell, whose own $HOME is the Windows profile.
$wslWork = '$HOME/uc-build'
$fetch = "mkdir -p $wslWork/_tools && cd $wslWork/_tools && for f in " +
($TOOL_FILES -join " ") +
"; do curl -fsSL -o `$f `"$($TOOLS_URL.TrimEnd('/'))/`$f`" || echo `" [!] optional/absent: `$f`"; done"
& wsl.exe -d $Distro -u root -- bash -lc "$fetch"
$run = "cd $wslWork/_tools && HF_TOKEN='$HF_TOKEN' TOOLS_URL='$TOOLS_URL' " +
($(if ($DEST_PREFIX) { "DEST_PREFIX='$DEST_PREFIX' " } else { "" })) +
"WORKDIR=$wslWork python3 run_matrix.py"
& wsl.exe -d $Distro -u root -- bash -lc "$run"
$wslRc = $LASTEXITCODE
if ($wslRc -eq 0) { Ok "linux_x64 lane complete - every model uploaded" }
else { Warn ("linux_x64 lane: {0} model(s) FAILED" -f $wslRc) }
}
}
# -- 5. report -------------------------------------------------------------------------
Head "RESULT - $gpu / $SM"
if ($winRc -ne $null) {
if ($winRc -eq 0) { Write-Host " win_x64 : OK (all models)" -ForegroundColor Green }
else { Write-Host (" win_x64 : {0} model(s) FAILED" -f $winRc) -ForegroundColor Red }
} else { Write-Host " win_x64 : not run (-Stage $Stage)" -ForegroundColor DarkGray }
if ($wslSkipReason) {
# An OS that cannot run WSL2 at all is a FACT about the image, not a failure
# of this run - Windows Server 2019 (17763) is below the 19041 floor and no
# amount of retrying changes that. Colour and exit code below treat the two
# kinds of skip differently, because a caller that branches on the exit code
# would otherwise see every single Server 2019 build as a failure.
$expectedSkip = $wslSkipReason -match "19041|WSL2 unavailable"
$c = if ($expectedSkip) { "DarkGray" } else { "Yellow" }
Write-Host (" linux_x64 : SKIPPED{0}" -f $(if ($expectedSkip) { " (expected on this Windows version)" } else { "" })) -ForegroundColor $c
Write-Host (" {0}" -f $wslSkipReason) -ForegroundColor $c
Write-Host " -> build these cells on a separate Linux GPU VM with bootstrap_linux.sh" -ForegroundColor $c
} elseif ($wslRc -ne $null) {
if ($wslRc -eq 0) { Write-Host " linux_x64 : OK (all models)" -ForegroundColor Green }
else { Write-Host (" linux_x64 : {0} model(s) FAILED" -f $wslRc) -ForegroundColor Red }
} else { Write-Host " linux_x64 : not run (-Stage $Stage)" -ForegroundColor DarkGray }
Write-Host ""
Write-Host " Verify what actually landed:" -ForegroundColor DarkGray
Write-Host " https://huggingface.co/faxenoff/code-daemon-embed-v1/tree/main" -ForegroundColor DarkGray
Write-Host (" every name must end in _{0}.engine for this box" -f $SM) -ForegroundColor DarkGray
$fail = 0
if ($winRc) { $fail += $winRc }
if ($wslRc) { $fail += $wslRc }
# A skip the OS makes unavoidable is not counted. A skip caused by something
# fixable - TCC driver mode, nested virtualization off, an install that failed -
# still is, because those are worth acting on and the operator may only be
# watching the exit code.
if ($wslSkipReason -and -not ($wslSkipReason -match "19041|WSL2 unavailable")) { $fail += 1 }
exit $fail