# bootstrap_win.ps1 - turn a bare Windows GPU VM into a two-OS TRT engine builder. # # One rented Windows box can produce BOTH halves of a {win_x64, linux_x64} pair: # the Windows engines natively, the Linux engines inside WSL2 against the same # physical GPU. That halves the rental - and, more importantly, it guarantees the # two halves saw the identical GPU, driver and ONNX, so a difference between them # is a real difference and not a difference between two rented machines. # # The WSL half is BEST EFFORT and says so loudly. GPU passthrough into WSL needs a # WDDM-mode driver and nested virtualization on the hypervisor; a Tesla card left # in TCC mode or a VM without nested virt will not have it, and this script # diagnoses which of those it is rather than reporting a mysterious skip. # # -- On-VM usage (the only command you type) ------------------------------------------- # $env:TOOLS_URL = "https://.s3.twcstorage.ru/_build-tools" # $env:HF_TOKEN = "hf_...write..." # irm "$($env:TOOLS_URL)/bootstrap_win.ps1" | iex # # Everything else is discovered: GPU arch, which models to build, at what precision. # # To pass a switch through the pipe, `iex` is not enough - it takes no arguments: # & ([scriptblock]::Create((irm "$($env:TOOLS_URL)/bootstrap_win.ps1"))) -Stage wsl # # -Stage win Windows lane only (skip WSL entirely) # -Stage wsl WSL lane only (assumes the Windows lane already ran) # -Stage all both (default) # -NoReboot never schedule a reboot; report what WSL still needs instead # -Distro WSL distribution (default Ubuntu-24.04) [CmdletBinding()] param( [ValidateSet("all", "win", "wsl")] [string] $Stage = "all", [switch] $NoReboot, [string] $Distro = "Ubuntu-24.04" ) $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" # Invoke-WebRequest is ~10x faster without it # -- output helpers -------------------------------------------------------------------- function Head($m) { Write-Host ""; Write-Host ("=" * 72) -ForegroundColor DarkGray; Write-Host " $m" -ForegroundColor Cyan; Write-Host ("=" * 72) -ForegroundColor DarkGray } function Step($m) { Write-Host "`n[>] $m" -ForegroundColor Cyan } function Ok($m) { Write-Host " [OK] $m" -ForegroundColor Green } function Warn($m) { Write-Host " [!] $m" -ForegroundColor Yellow } function Die($m) { Write-Host " [!!] $m" -ForegroundColor Red; exit 1 } $TOOLS_URL = [Environment]::GetEnvironmentVariable("TOOLS_URL") $HF_TOKEN = [Environment]::GetEnvironmentVariable("HF_TOKEN") $DEST_PREFIX = [Environment]::GetEnvironmentVariable("DEST_PREFIX") $WORK = Join-Path $env:USERPROFILE "uc-build" $TOOLS = Join-Path $WORK "_tools" $STATE = Join-Path $WORK "bootstrap-state.json" # The files a build needs on the box. build_matrix.json is what makes the run # self-describing: precision per model per arch comes from the catalog, not from # whatever the operator remembered to export. $TOOL_FILES = @( "run_matrix.py", "build_matrix.json", "build_engines.ps1", "build_engines.sh", "build_trt_engine.py", "quantize_onnx.py", "onnx_fp16.py", "calib_corpus.jsonl" ) # Missing one of these is not fatal: fp16-only archs never touch the quantizer or # the corpus, and paying for a 404 there would block a Turing run for no reason. $TOOL_OPTIONAL = @("quantize_onnx.py", "calib_corpus.jsonl") Head "ultracode cloud builder - Windows VM bootstrap" if (-not $TOOLS_URL) { Die "set TOOLS_URL first, e.g. `$env:TOOLS_URL='https://.s3.twcstorage.ru/_build-tools'" } if (-not $HF_TOKEN -and -not $DEST_PREFIX) { Die "set HF_TOKEN (a WRITE token) or DEST_PREFIX - otherwise the engines are built and discarded." } # -- 0. preflight: is this machine even eligible? -------------------------------------- Step "Preflight" $isAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent() ).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) if (-not $isAdmin) { Die "run this in an ELEVATED PowerShell - installing Python and WSL needs it." } Ok "elevated" $osv = [Environment]::OSVersion.Version $caption = (Get-CimInstance Win32_OperatingSystem).Caption Ok ("{0} (build {1})" -f $caption, $osv.Build) if ($osv.Build -lt 19041) { # Server 2019 is 17763 and is what Selectel offers, so this is the normal # case there rather than a problem: win_x64 builds here, linux_x64 comes from # a separate Ubuntu VM. Stated once, plainly, so it does not read as a fault. Write-Host " [i] build < 19041 - no WSL2 on this image. win_x64 builds here;" -ForegroundColor DarkGray Write-Host " linux_x64 comes from a Linux VM (bootstrap_linux.sh). Expected on Server 2019." -ForegroundColor DarkGray } if (-not (Get-Command nvidia-smi -ErrorAction SilentlyContinue)) { Die "nvidia-smi not on PATH - install the NVIDIA driver first. On a fresh cloud VM the driver is often NOT preinstalled; ask the provider for a GPU image or install from nvidia.com." } $cc = (& nvidia-smi --query-gpu=compute_cap --format=csv,noheader | Select-Object -First 1).Trim() $gpu = (& nvidia-smi --query-gpu=name --format=csv,noheader | Select-Object -First 1).Trim() $drv = (& nvidia-smi --query-gpu=driver_version --format=csv,noheader | Select-Object -First 1).Trim() if (-not $cc) { Die "nvidia-smi reported no compute capability - the driver is present but the GPU is not usable." } $SM = "sm_" + ($cc -replace '\.', '') Ok ("GPU: {0} compute_cap {1} -> {2} driver {3}" -f $gpu, $cc, $SM, $drv) # The driver model decides whether WSL can see this GPU at all. TCC (Tesla Compute # Cluster) has no WDDM display stack, and /dev/dxg - the whole basis of WSL GPU # passthrough - is a WDDM construct. Read it now so the WSL stage can explain # itself instead of just failing. $driverModel = "" try { $dm = (& nvidia-smi --query-gpu=driver_model.current --format=csv,noheader 2>$null | Select-Object -First 1) if ($dm) { $driverModel = $dm.Trim() } } catch { } if ($driverModel) { Ok ("driver model: {0}" -f $driverModel) } # -- 1. Python 3.13 - the cu13 TensorRT wheels are built for it ------------------------ Step "Python" function Find-Py313 { # Return the FULL PATH, never the bare name "py". # # PowerShell resolves a command as alias -> FUNCTION -> cmdlet -> program, and # is case-insensitive. With `$PyExe = "py"` and a helper function named `Py` # in scope, `& $PyExe` calls the FUNCTION, which calls `& $PyExe` again: # infinite recursion, ending in "The script failed due to call depth # overflow" right after the Python step announces itself. A path can never # resolve to a function. $launcher = Get-Command py.exe -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1 if ($launcher) { & $launcher.Source -3.13 -c "import sys" *> $null if ($LASTEXITCODE -eq 0) { return @{ Exe = $launcher.Source; Pre = @("-3.13") } } } $direct = Join-Path $env:LOCALAPPDATA "Programs\Python\Python313\python.exe" if (Test-Path $direct) { return @{ Exe = $direct; Pre = @() } } $direct = Join-Path $env:ProgramFiles "Python313\python.exe" if (Test-Path $direct) { return @{ Exe = $direct; Pre = @() } } return $null } $py = Find-Py313 if (-not $py) { Warn "Python 3.13 not found - installing (the cu13 tensorrt wheels have no 3.12/3.11 build)" $installer = Join-Path $env:TEMP "python-3.13-amd64.exe" # python.org direct, not winget: winget is absent on Server images and its # Store-backed Python is a stub that answers `python` with a store page. $url = "https://www.python.org/ftp/python/3.13.7/python-3.13.7-amd64.exe" Invoke-WebRequest -Uri $url -OutFile $installer -UseBasicParsing # PrependPath so the WSL side and any later shell find it too; Include_launcher # gives us `py -3.13`, which is what build_engines.ps1 looks for. $p = Start-Process -FilePath $installer -Wait -PassThru -ArgumentList @( "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_launcher=1", "Include_pip=1" ) if ($p.ExitCode -ne 0) { Die ("python installer exited {0}" -f $p.ExitCode) } $env:Path = [Environment]::GetEnvironmentVariable("Path", "Machine") + ";" + [Environment]::GetEnvironmentVariable("Path", "User") $py = Find-Py313 if (-not $py) { Die "Python 3.13 still not found after install." } } $PyExe = $py.Exe; $PyPre = $py.Pre function Invoke-Py { & $PyExe @PyPre @args; if ($LASTEXITCODE -ne 0) { throw ("python exited {0}" -f $LASTEXITCODE) } } Ok ("python: {0}" -f ((& $PyExe @PyPre --version) 2>&1)) # -- 2. fetch the build tools ---------------------------------------------------------- Step "Fetching build tools from $TOOLS_URL" New-Item -ItemType Directory -Force -Path $WORK, $TOOLS | Out-Null $fetched = @(); $missing = @() foreach ($f in $TOOL_FILES) { $dst = Join-Path $TOOLS $f try { Invoke-WebRequest -Uri ("{0}/{1}" -f $TOOLS_URL.TrimEnd('/'), $f) -OutFile $dst -UseBasicParsing $fetched += $f } catch { if ($TOOL_OPTIONAL -contains $f) { $missing += $f } else { Die ("could not fetch {0} from {1} - is the prefix public? ({2})" -f $f, $TOOLS_URL, $_.Exception.Message) } } } Ok ("fetched: {0}" -f ($fetched -join ", ")) if ($missing) { Warn ("absent (fine unless an INT8 arch needs PTQ): {0}" -f ($missing -join ", ")) } # -- 3. the Windows lane --------------------------------------------------------------- $winRc = $null if ($Stage -eq "all" -or $Stage -eq "win") { Head "Windows lane - building win_x64 engines on $SM" $env:TOOLS_URL = $TOOLS_URL $env:WORKDIR = $WORK Push-Location $TOOLS try { & $PyExe @PyPre "run_matrix.py" $winRc = $LASTEXITCODE } finally { Pop-Location } if ($winRc -eq 0) { Ok "win_x64 lane complete - every model uploaded" } else { Warn ("win_x64 lane: {0} model(s) FAILED - see the log above" -f $winRc) } } # -- 4. the WSL lane ------------------------------------------------------------------- function Test-WslFeatures { <# Both features must be Enabled before a distro can run in WSL2 mode. Reported separately because the failure modes differ: no VirtualMachinePlatform usually means nested virtualization is off on the hypervisor, which is a provider-side setting no amount of local work fixes. #> $res = @{} foreach ($n in @("Microsoft-Windows-Subsystem-Linux", "VirtualMachinePlatform")) { try { $res[$n] = (Get-WindowsOptionalFeature -Online -FeatureName $n).State } catch { $res[$n] = "Unknown" } } return $res } function Enable-WslFeatures { $needReboot = $false foreach ($n in @("Microsoft-Windows-Subsystem-Linux", "VirtualMachinePlatform")) { $st = (Get-WindowsOptionalFeature -Online -FeatureName $n).State if ($st -ne "Enabled") { Step "Enabling Windows feature $n" $r = Enable-WindowsOptionalFeature -Online -FeatureName $n -All -NoRestart if ($r.RestartNeeded) { $needReboot = $true } } } return $needReboot } function Register-Resume { <# A RunOnce entry, not a scheduled task: RunOnce fires on the next interactive logon of THIS user with the console attached, which is what an operator reconnecting over RDP actually gets. A SYSTEM scheduled task would run headless and its output would go nowhere. The token does NOT go in the registry. A RunOnce value is world-readable and survives in registry backups and any image the provider snapshots; a file with an Administrators-only ACL, deleted by the resume script as its first act, keeps the blast radius to the reboot window. #> $tokFile = Join-Path $WORK "resume-token.txt" Set-Content -Path $tokFile -Value $HF_TOKEN -Encoding ascii -NoNewline $acl = Get-Acl $tokFile $acl.SetAccessRuleProtection($true, $false) # drop inherited user-readable ACEs $acl.SetAccessRule((New-Object System.Security.AccessControl.FileSystemAccessRule( "BUILTIN\Administrators", "FullControl", "Allow"))) Set-Acl -Path $tokFile -AclObject $acl # `iex` takes no -Args; a scriptblock does. This is the only shape that can # pass -Stage through a piped one-liner. $inner = ("`$env:TOOLS_URL='{0}'; " + "`$env:HF_TOKEN=(Get-Content '{1}' -Raw); Remove-Item '{1}' -Force; " + "& ([scriptblock]::Create((irm '{0}/bootstrap_win.ps1'))) -Stage wsl") -f $TOOLS_URL, $tokFile $enc = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($inner)) $cmd = "powershell -NoProfile -ExecutionPolicy Bypass -EncodedCommand $enc" New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\RunOnce" ` -Name "UltracodeBuildResume" -Value $cmd -PropertyType String -Force | Out-Null Warn "the WSL lane will resume automatically at the next logon (RunOnce)" Warn ("token parked at {0} (Administrators only); the resume deletes it before building" -f $tokFile) } $wslRc = $null $wslSkipReason = $null if ($Stage -eq "all" -or $Stage -eq "wsl") { Head "WSL lane - building linux_x64 engines on the same $SM GPU" if ($osv.Build -lt 19041) { $wslSkipReason = "Windows build $($osv.Build) < 19041 - WSL2 unavailable on this image" } else { $feat = Test-WslFeatures foreach ($k in $feat.Keys) { Write-Host (" feature {0,-38} {1}" -f $k, $feat[$k]) } if ($feat["VirtualMachinePlatform"] -ne "Enabled" -or $feat["Microsoft-Windows-Subsystem-Linux"] -ne "Enabled") { if ($NoReboot) { $wslSkipReason = "WSL features are off and -NoReboot was given. Enable them and rerun with -Stage wsl." } else { $need = Enable-WslFeatures if ($need) { Register-Resume Head "REBOOT REQUIRED" Write-Host " Windows features were just enabled. The machine reboots in 15 seconds;" Write-Host " log back in and the WSL lane resumes on its own." if ($winRc -ne $null -and $winRc -ne 0) { Warn ("the Windows lane had {0} failure(s) - re-run -Stage win after the reboot" -f $winRc) } Start-Sleep -Seconds 15 Restart-Computer -Force exit 0 } } } } if (-not $wslSkipReason) { # Is a distro registered? `wsl -l -q` output is UTF-16 on older builds - # match loosely rather than parsing it as a clean list. $distros = (& wsl.exe -l -q 2>$null) -join "`n" $haveDistro = $distros -match [regex]::Escape($Distro.Split('-')[0]) if (-not $haveDistro) { Step "Installing WSL distribution $Distro" # --no-launch registers without dropping into the interactive # "create a user" prompt, which would hang an unattended run forever. # --web-download avoids the Microsoft Store, absent on Server images. & wsl.exe --install --no-launch --web-download -d $Distro if ($LASTEXITCODE -ne 0) { Warn "wsl --install -d $Distro failed; retrying without --web-download" & wsl.exe --install --no-launch -d $Distro } if ($LASTEXITCODE -ne 0) { $wslSkipReason = "could not install the $Distro distribution (wsl --install exited $LASTEXITCODE)" } else { # First run finishes registration. -u root skips user creation. & wsl.exe -d $Distro -u root -- true 2>$null | Out-Null } } } if (-not $wslSkipReason) { Step "Probing the GPU inside WSL" $wslSmi = (& wsl.exe -d $Distro -u root -- bash -lc "nvidia-smi --query-gpu=name,compute_cap --format=csv,noheader 2>&1") -join " " if ($LASTEXITCODE -ne 0 -or $wslSmi -notmatch '\d\.\d') { # This is the failure worth diagnosing precisely - it is the single # most likely reason the Linux half of the matrix does not get built. Warn "nvidia-smi inside WSL did not report a GPU." Write-Host " raw: $wslSmi" -ForegroundColor DarkGray Write-Host "" Write-Host " Why this happens, in order of likelihood on a cloud VM:" -ForegroundColor Yellow if ($driverModel -match "TCC") { Write-Host " 1. THIS ONE: the driver is in TCC mode ($driverModel). WSL needs WDDM." -ForegroundColor Red Write-Host " Try: nvidia-smi -dm 0 then reboot, then rerun with -Stage wsl" Write-Host " (Datacenter cards default to TCC; some vGPU profiles cannot switch.)" } else { Write-Host " 1. Driver model must be WDDM, not TCC. Check: nvidia-smi -q | findstr /i `"Driver Model`"" Write-Host " Switch with: nvidia-smi -dm 0 (then reboot)" } Write-Host " 2. NVIDIA vGPU / GRID profiles do not expose /dev/dxg to WSL at all." Write-Host " If this is a vGPU VM, the Linux engines need a separate Linux VM." Write-Host " 3. Nested virtualization off on the hypervisor. In VMware/Cloud Director:" Write-Host " the VM needs 'Expose hardware assisted virtualization' (nested_hypervisor_enabled)." Write-Host " 4. Driver older than 465 - no WSL CUDA support at all. This one is $drv." Write-Host "" $wslSkipReason = "no GPU visible inside WSL (see the diagnosis above)" } else { Ok ("WSL sees: {0}" -f $wslSmi.Trim()) } } if (-not $wslSkipReason) { Step "Preparing the WSL toolchain" # python3-pip + venv only. TensorRT arrives as a pip wheel from # build_trt_engine.py, and CUDA itself comes from the Windows driver via # /usr/lib/wsl/lib - installing a CUDA toolkit in the distro would only # shadow it with a second, wrong libcuda. $prep = @( "export DEBIAN_FRONTEND=noninteractive", "apt-get update -qq", "apt-get install -y -qq python3 python3-pip python3-venv curl ca-certificates >/dev/null", "python3 -m pip install --break-system-packages -q -U pip huggingface_hub 2>/dev/null || python3 -m pip install -q -U pip huggingface_hub" ) -join " && " & wsl.exe -d $Distro -u root -- bash -lc "$prep" if ($LASTEXITCODE -ne 0) { Warn "apt/pip prep returned $LASTEXITCODE - continuing, the build may still work" } Step "Running the Linux matrix inside WSL" # The tools are re-fetched INSIDE the distro rather than read across # /mnt/c: DrvFs has no executable bit and its inode semantics confuse # anything that stats a file twice. A second download costs seconds. # Single-quoted: `$HOME` is expanded by bash inside the distro (root's home # there), never by PowerShell, whose own $HOME is the Windows profile. $wslWork = '$HOME/uc-build' $fetch = "mkdir -p $wslWork/_tools && cd $wslWork/_tools && for f in " + ($TOOL_FILES -join " ") + "; do curl -fsSL -o `$f `"$($TOOLS_URL.TrimEnd('/'))/`$f`" || echo `" [!] optional/absent: `$f`"; done" & wsl.exe -d $Distro -u root -- bash -lc "$fetch" $run = "cd $wslWork/_tools && HF_TOKEN='$HF_TOKEN' TOOLS_URL='$TOOLS_URL' " + ($(if ($DEST_PREFIX) { "DEST_PREFIX='$DEST_PREFIX' " } else { "" })) + "WORKDIR=$wslWork python3 run_matrix.py" & wsl.exe -d $Distro -u root -- bash -lc "$run" $wslRc = $LASTEXITCODE if ($wslRc -eq 0) { Ok "linux_x64 lane complete - every model uploaded" } else { Warn ("linux_x64 lane: {0} model(s) FAILED" -f $wslRc) } } } # -- 5. report ------------------------------------------------------------------------- Head "RESULT - $gpu / $SM" if ($winRc -ne $null) { if ($winRc -eq 0) { Write-Host " win_x64 : OK (all models)" -ForegroundColor Green } else { Write-Host (" win_x64 : {0} model(s) FAILED" -f $winRc) -ForegroundColor Red } } else { Write-Host " win_x64 : not run (-Stage $Stage)" -ForegroundColor DarkGray } if ($wslSkipReason) { # An OS that cannot run WSL2 at all is a FACT about the image, not a failure # of this run - Windows Server 2019 (17763) is below the 19041 floor and no # amount of retrying changes that. Colour and exit code below treat the two # kinds of skip differently, because a caller that branches on the exit code # would otherwise see every single Server 2019 build as a failure. $expectedSkip = $wslSkipReason -match "19041|WSL2 unavailable" $c = if ($expectedSkip) { "DarkGray" } else { "Yellow" } Write-Host (" linux_x64 : SKIPPED{0}" -f $(if ($expectedSkip) { " (expected on this Windows version)" } else { "" })) -ForegroundColor $c Write-Host (" {0}" -f $wslSkipReason) -ForegroundColor $c Write-Host " -> build these cells on a separate Linux GPU VM with bootstrap_linux.sh" -ForegroundColor $c } elseif ($wslRc -ne $null) { if ($wslRc -eq 0) { Write-Host " linux_x64 : OK (all models)" -ForegroundColor Green } else { Write-Host (" linux_x64 : {0} model(s) FAILED" -f $wslRc) -ForegroundColor Red } } else { Write-Host " linux_x64 : not run (-Stage $Stage)" -ForegroundColor DarkGray } Write-Host "" Write-Host " Verify what actually landed:" -ForegroundColor DarkGray Write-Host " https://huggingface.co/faxenoff/code-daemon-embed-v1/tree/main" -ForegroundColor DarkGray Write-Host (" every name must end in _{0}.engine for this box" -f $SM) -ForegroundColor DarkGray $fail = 0 if ($winRc) { $fail += $winRc } if ($wslRc) { $fail += $wslRc } # A skip the OS makes unavoidable is not counted. A skip caused by something # fixable - TCC driver mode, nested virtualization off, an install that failed - # still is, because those are worth acting on and the operator may only be # watching the exit code. if ($wslSkipReason -and -not ($wslSkipReason -match "19041|WSL2 unavailable")) { $fail += 1 } exit $fail