File size: 22,360 Bytes
40da26e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
# bootstrap_win.ps1 - turn a bare Windows GPU VM into a two-OS TRT engine builder.
#
# One rented Windows box can produce BOTH halves of a {win_x64, linux_x64} pair:
# the Windows engines natively, the Linux engines inside WSL2 against the same
# physical GPU. That halves the rental - and, more importantly, it guarantees the
# two halves saw the identical GPU, driver and ONNX, so a difference between them
# is a real difference and not a difference between two rented machines.
#
# The WSL half is BEST EFFORT and says so loudly. GPU passthrough into WSL needs a
# WDDM-mode driver and nested virtualization on the hypervisor; a Tesla card left
# in TCC mode or a VM without nested virt will not have it, and this script
# diagnoses which of those it is rather than reporting a mysterious skip.
#
# -- On-VM usage (the only command you type) -------------------------------------------
#   $env:TOOLS_URL = "https://<bucket>.s3.twcstorage.ru/_build-tools"
#   $env:HF_TOKEN  = "hf_...write..."
#   irm "$($env:TOOLS_URL)/bootstrap_win.ps1" | iex
#
# Everything else is discovered: GPU arch, which models to build, at what precision.
#
# To pass a switch through the pipe, `iex` is not enough - it takes no arguments:
#   & ([scriptblock]::Create((irm "$($env:TOOLS_URL)/bootstrap_win.ps1"))) -Stage wsl
#
#   -Stage win      Windows lane only (skip WSL entirely)
#   -Stage wsl      WSL lane only (assumes the Windows lane already ran)
#   -Stage all      both (default)
#   -NoReboot       never schedule a reboot; report what WSL still needs instead
#   -Distro         WSL distribution (default Ubuntu-24.04)
[CmdletBinding()]
param(
    [ValidateSet("all", "win", "wsl")] [string] $Stage = "all",
    [switch] $NoReboot,
    [string] $Distro = "Ubuntu-24.04"
)

$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"   # Invoke-WebRequest is ~10x faster without it

# -- output helpers --------------------------------------------------------------------
function Head($m) { Write-Host ""; Write-Host ("=" * 72) -ForegroundColor DarkGray; Write-Host " $m" -ForegroundColor Cyan; Write-Host ("=" * 72) -ForegroundColor DarkGray }
function Step($m) { Write-Host "`n[>] $m" -ForegroundColor Cyan }
function Ok($m)   { Write-Host "  [OK] $m" -ForegroundColor Green }
function Warn($m) { Write-Host "  [!]  $m" -ForegroundColor Yellow }
function Die($m)  { Write-Host "  [!!] $m" -ForegroundColor Red; exit 1 }

$TOOLS_URL = [Environment]::GetEnvironmentVariable("TOOLS_URL")
$HF_TOKEN  = [Environment]::GetEnvironmentVariable("HF_TOKEN")
$DEST_PREFIX = [Environment]::GetEnvironmentVariable("DEST_PREFIX")
$WORK      = Join-Path $env:USERPROFILE "uc-build"
$TOOLS     = Join-Path $WORK "_tools"
$STATE     = Join-Path $WORK "bootstrap-state.json"

# The files a build needs on the box. build_matrix.json is what makes the run
# self-describing: precision per model per arch comes from the catalog, not from
# whatever the operator remembered to export.
$TOOL_FILES = @(
    "run_matrix.py", "build_matrix.json",
    "build_engines.ps1", "build_engines.sh",
    "build_trt_engine.py", "quantize_onnx.py", "onnx_fp16.py", "calib_corpus.jsonl"
)
# Missing one of these is not fatal: fp16-only archs never touch the quantizer or
# the corpus, and paying for a 404 there would block a Turing run for no reason.
$TOOL_OPTIONAL = @("quantize_onnx.py", "calib_corpus.jsonl")

Head "ultracode cloud builder - Windows VM bootstrap"

if (-not $TOOLS_URL) { Die "set TOOLS_URL first, e.g. `$env:TOOLS_URL='https://<bucket>.s3.twcstorage.ru/_build-tools'" }
if (-not $HF_TOKEN -and -not $DEST_PREFIX) {
    Die "set HF_TOKEN (a WRITE token) or DEST_PREFIX - otherwise the engines are built and discarded."
}

# -- 0. preflight: is this machine even eligible? --------------------------------------
Step "Preflight"

$isAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()
           ).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
if (-not $isAdmin) { Die "run this in an ELEVATED PowerShell - installing Python and WSL needs it." }
Ok "elevated"

$osv = [Environment]::OSVersion.Version
$caption = (Get-CimInstance Win32_OperatingSystem).Caption
Ok ("{0} (build {1})" -f $caption, $osv.Build)
if ($osv.Build -lt 19041) {
    # Server 2019 is 17763 and is what Selectel offers, so this is the normal
    # case there rather than a problem: win_x64 builds here, linux_x64 comes from
    # a separate Ubuntu VM. Stated once, plainly, so it does not read as a fault.
    Write-Host "  [i]  build < 19041 - no WSL2 on this image. win_x64 builds here;" -ForegroundColor DarkGray
    Write-Host "       linux_x64 comes from a Linux VM (bootstrap_linux.sh). Expected on Server 2019." -ForegroundColor DarkGray
}

if (-not (Get-Command nvidia-smi -ErrorAction SilentlyContinue)) {
    Die "nvidia-smi not on PATH - install the NVIDIA driver first. On a fresh cloud VM the driver is often NOT preinstalled; ask the provider for a GPU image or install from nvidia.com."
}
$cc  = (& nvidia-smi --query-gpu=compute_cap --format=csv,noheader | Select-Object -First 1).Trim()
$gpu = (& nvidia-smi --query-gpu=name --format=csv,noheader | Select-Object -First 1).Trim()
$drv = (& nvidia-smi --query-gpu=driver_version --format=csv,noheader | Select-Object -First 1).Trim()
if (-not $cc) { Die "nvidia-smi reported no compute capability - the driver is present but the GPU is not usable." }
$SM = "sm_" + ($cc -replace '\.', '')
Ok ("GPU: {0}   compute_cap {1} -> {2}   driver {3}" -f $gpu, $cc, $SM, $drv)

# The driver model decides whether WSL can see this GPU at all. TCC (Tesla Compute
# Cluster) has no WDDM display stack, and /dev/dxg - the whole basis of WSL GPU
# passthrough - is a WDDM construct. Read it now so the WSL stage can explain
# itself instead of just failing.
$driverModel = ""
try {
    $dm = (& nvidia-smi --query-gpu=driver_model.current --format=csv,noheader 2>$null | Select-Object -First 1)
    if ($dm) { $driverModel = $dm.Trim() }
} catch { }
if ($driverModel) { Ok ("driver model: {0}" -f $driverModel) }

# -- 1. Python 3.13 - the cu13 TensorRT wheels are built for it ------------------------
Step "Python"

function Find-Py313 {
    # Return the FULL PATH, never the bare name "py".
    #
    # PowerShell resolves a command as alias -> FUNCTION -> cmdlet -> program, and
    # is case-insensitive. With `$PyExe = "py"` and a helper function named `Py`
    # in scope, `& $PyExe` calls the FUNCTION, which calls `& $PyExe` again:
    # infinite recursion, ending in "The script failed due to call depth
    # overflow" right after the Python step announces itself. A path can never
    # resolve to a function.
    $launcher = Get-Command py.exe -CommandType Application -ErrorAction SilentlyContinue |
                Select-Object -First 1
    if ($launcher) {
        & $launcher.Source -3.13 -c "import sys" *> $null
        if ($LASTEXITCODE -eq 0) { return @{ Exe = $launcher.Source; Pre = @("-3.13") } }
    }
    $direct = Join-Path $env:LOCALAPPDATA "Programs\Python\Python313\python.exe"
    if (Test-Path $direct) { return @{ Exe = $direct; Pre = @() } }
    $direct = Join-Path $env:ProgramFiles "Python313\python.exe"
    if (Test-Path $direct) { return @{ Exe = $direct; Pre = @() } }
    return $null
}

$py = Find-Py313
if (-not $py) {
    Warn "Python 3.13 not found - installing (the cu13 tensorrt wheels have no 3.12/3.11 build)"
    $installer = Join-Path $env:TEMP "python-3.13-amd64.exe"
    # python.org direct, not winget: winget is absent on Server images and its
    # Store-backed Python is a stub that answers `python` with a store page.
    $url = "https://www.python.org/ftp/python/3.13.7/python-3.13.7-amd64.exe"
    Invoke-WebRequest -Uri $url -OutFile $installer -UseBasicParsing
    # PrependPath so the WSL side and any later shell find it too; Include_launcher
    # gives us `py -3.13`, which is what build_engines.ps1 looks for.
    $p = Start-Process -FilePath $installer -Wait -PassThru -ArgumentList @(
        "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_launcher=1", "Include_pip=1"
    )
    if ($p.ExitCode -ne 0) { Die ("python installer exited {0}" -f $p.ExitCode) }
    $env:Path = [Environment]::GetEnvironmentVariable("Path", "Machine") + ";" +
                [Environment]::GetEnvironmentVariable("Path", "User")
    $py = Find-Py313
    if (-not $py) { Die "Python 3.13 still not found after install." }
}
$PyExe = $py.Exe; $PyPre = $py.Pre
function Invoke-Py { & $PyExe @PyPre @args; if ($LASTEXITCODE -ne 0) { throw ("python exited {0}" -f $LASTEXITCODE) } }
Ok ("python: {0}" -f ((& $PyExe @PyPre --version) 2>&1))

# -- 2. fetch the build tools ----------------------------------------------------------
Step "Fetching build tools from $TOOLS_URL"
New-Item -ItemType Directory -Force -Path $WORK, $TOOLS | Out-Null
$fetched = @(); $missing = @()
foreach ($f in $TOOL_FILES) {
    $dst = Join-Path $TOOLS $f
    try {
        Invoke-WebRequest -Uri ("{0}/{1}" -f $TOOLS_URL.TrimEnd('/'), $f) -OutFile $dst -UseBasicParsing
        $fetched += $f
    } catch {
        if ($TOOL_OPTIONAL -contains $f) { $missing += $f } else { Die ("could not fetch {0} from {1} - is the prefix public? ({2})" -f $f, $TOOLS_URL, $_.Exception.Message) }
    }
}
Ok ("fetched: {0}" -f ($fetched -join ", "))
if ($missing) { Warn ("absent (fine unless an INT8 arch needs PTQ): {0}" -f ($missing -join ", ")) }

# -- 3. the Windows lane ---------------------------------------------------------------
$winRc = $null
if ($Stage -eq "all" -or $Stage -eq "win") {
    Head "Windows lane - building win_x64 engines on $SM"
    $env:TOOLS_URL = $TOOLS_URL
    $env:WORKDIR   = $WORK
    Push-Location $TOOLS
    try {
        & $PyExe @PyPre "run_matrix.py"
        $winRc = $LASTEXITCODE
    } finally { Pop-Location }
    if ($winRc -eq 0) { Ok "win_x64 lane complete - every model uploaded" }
    else { Warn ("win_x64 lane: {0} model(s) FAILED - see the log above" -f $winRc) }
}

# -- 4. the WSL lane -------------------------------------------------------------------
function Test-WslFeatures {
    <#  Both features must be Enabled before a distro can run in WSL2 mode.
        Reported separately because the failure modes differ: no VirtualMachinePlatform
        usually means nested virtualization is off on the hypervisor, which is a
        provider-side setting no amount of local work fixes. #>
    $res = @{}
    foreach ($n in @("Microsoft-Windows-Subsystem-Linux", "VirtualMachinePlatform")) {
        try { $res[$n] = (Get-WindowsOptionalFeature -Online -FeatureName $n).State }
        catch { $res[$n] = "Unknown" }
    }
    return $res
}

function Enable-WslFeatures {
    $needReboot = $false
    foreach ($n in @("Microsoft-Windows-Subsystem-Linux", "VirtualMachinePlatform")) {
        $st = (Get-WindowsOptionalFeature -Online -FeatureName $n).State
        if ($st -ne "Enabled") {
            Step "Enabling Windows feature $n"
            $r = Enable-WindowsOptionalFeature -Online -FeatureName $n -All -NoRestart
            if ($r.RestartNeeded) { $needReboot = $true }
        }
    }
    return $needReboot
}

function Register-Resume {
    <#  A RunOnce entry, not a scheduled task: RunOnce fires on the next
        interactive logon of THIS user with the console attached, which is what an
        operator reconnecting over RDP actually gets. A SYSTEM scheduled task
        would run headless and its output would go nowhere.

        The token does NOT go in the registry. A RunOnce value is world-readable
        and survives in registry backups and any image the provider snapshots; a
        file with an Administrators-only ACL, deleted by the resume script as its
        first act, keeps the blast radius to the reboot window. #>
    $tokFile = Join-Path $WORK "resume-token.txt"
    Set-Content -Path $tokFile -Value $HF_TOKEN -Encoding ascii -NoNewline
    $acl = Get-Acl $tokFile
    $acl.SetAccessRuleProtection($true, $false)   # drop inherited user-readable ACEs
    $acl.SetAccessRule((New-Object System.Security.AccessControl.FileSystemAccessRule(
        "BUILTIN\Administrators", "FullControl", "Allow")))
    Set-Acl -Path $tokFile -AclObject $acl

    # `iex` takes no -Args; a scriptblock does. This is the only shape that can
    # pass -Stage through a piped one-liner.
    $inner = ("`$env:TOOLS_URL='{0}'; " +
              "`$env:HF_TOKEN=(Get-Content '{1}' -Raw); Remove-Item '{1}' -Force; " +
              "& ([scriptblock]::Create((irm '{0}/bootstrap_win.ps1'))) -Stage wsl") -f $TOOLS_URL, $tokFile
    $enc = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($inner))
    $cmd = "powershell -NoProfile -ExecutionPolicy Bypass -EncodedCommand $enc"

    New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\RunOnce" `
        -Name "UltracodeBuildResume" -Value $cmd -PropertyType String -Force | Out-Null
    Warn "the WSL lane will resume automatically at the next logon (RunOnce)"
    Warn ("token parked at {0} (Administrators only); the resume deletes it before building" -f $tokFile)
}

$wslRc = $null
$wslSkipReason = $null

if ($Stage -eq "all" -or $Stage -eq "wsl") {
    Head "WSL lane - building linux_x64 engines on the same $SM GPU"

    if ($osv.Build -lt 19041) {
        $wslSkipReason = "Windows build $($osv.Build) < 19041 - WSL2 unavailable on this image"
    } else {
        $feat = Test-WslFeatures
        foreach ($k in $feat.Keys) { Write-Host ("  feature {0,-38} {1}" -f $k, $feat[$k]) }

        if ($feat["VirtualMachinePlatform"] -ne "Enabled" -or
            $feat["Microsoft-Windows-Subsystem-Linux"] -ne "Enabled") {
            if ($NoReboot) {
                $wslSkipReason = "WSL features are off and -NoReboot was given. Enable them and rerun with -Stage wsl."
            } else {
                $need = Enable-WslFeatures
                if ($need) {
                    Register-Resume
                    Head "REBOOT REQUIRED"
                    Write-Host "  Windows features were just enabled. The machine reboots in 15 seconds;"
                    Write-Host "  log back in and the WSL lane resumes on its own."
                    if ($winRc -ne $null -and $winRc -ne 0) {
                        Warn ("the Windows lane had {0} failure(s) - re-run -Stage win after the reboot" -f $winRc)
                    }
                    Start-Sleep -Seconds 15
                    Restart-Computer -Force
                    exit 0
                }
            }
        }
    }

    if (-not $wslSkipReason) {
        # Is a distro registered? `wsl -l -q` output is UTF-16 on older builds -
        # match loosely rather than parsing it as a clean list.
        $distros = (& wsl.exe -l -q 2>$null) -join "`n"
        $haveDistro = $distros -match [regex]::Escape($Distro.Split('-')[0])

        if (-not $haveDistro) {
            Step "Installing WSL distribution $Distro"
            # --no-launch registers without dropping into the interactive
            # "create a user" prompt, which would hang an unattended run forever.
            # --web-download avoids the Microsoft Store, absent on Server images.
            & wsl.exe --install --no-launch --web-download -d $Distro
            if ($LASTEXITCODE -ne 0) {
                Warn "wsl --install -d $Distro failed; retrying without --web-download"
                & wsl.exe --install --no-launch -d $Distro
            }
            if ($LASTEXITCODE -ne 0) { $wslSkipReason = "could not install the $Distro distribution (wsl --install exited $LASTEXITCODE)" }
            else {
                # First run finishes registration. -u root skips user creation.
                & wsl.exe -d $Distro -u root -- true 2>$null | Out-Null
            }
        }
    }

    if (-not $wslSkipReason) {
        Step "Probing the GPU inside WSL"
        $wslSmi = (& wsl.exe -d $Distro -u root -- bash -lc "nvidia-smi --query-gpu=name,compute_cap --format=csv,noheader 2>&1") -join " "
        if ($LASTEXITCODE -ne 0 -or $wslSmi -notmatch '\d\.\d') {
            # This is the failure worth diagnosing precisely - it is the single
            # most likely reason the Linux half of the matrix does not get built.
            Warn "nvidia-smi inside WSL did not report a GPU."
            Write-Host "      raw: $wslSmi" -ForegroundColor DarkGray
            Write-Host ""
            Write-Host "  Why this happens, in order of likelihood on a cloud VM:" -ForegroundColor Yellow
            if ($driverModel -match "TCC") {
                Write-Host "   1. THIS ONE: the driver is in TCC mode ($driverModel). WSL needs WDDM." -ForegroundColor Red
                Write-Host "      Try:  nvidia-smi -dm 0     then reboot, then rerun with -Stage wsl"
                Write-Host "      (Datacenter cards default to TCC; some vGPU profiles cannot switch.)"
            } else {
                Write-Host "   1. Driver model must be WDDM, not TCC. Check: nvidia-smi -q | findstr /i `"Driver Model`""
                Write-Host "      Switch with: nvidia-smi -dm 0   (then reboot)"
            }
            Write-Host "   2. NVIDIA vGPU / GRID profiles do not expose /dev/dxg to WSL at all."
            Write-Host "      If this is a vGPU VM, the Linux engines need a separate Linux VM."
            Write-Host "   3. Nested virtualization off on the hypervisor. In VMware/Cloud Director:"
            Write-Host "      the VM needs 'Expose hardware assisted virtualization' (nested_hypervisor_enabled)."
            Write-Host "   4. Driver older than 465 - no WSL CUDA support at all. This one is $drv."
            Write-Host ""
            $wslSkipReason = "no GPU visible inside WSL (see the diagnosis above)"
        } else {
            Ok ("WSL sees: {0}" -f $wslSmi.Trim())
        }
    }

    if (-not $wslSkipReason) {
        Step "Preparing the WSL toolchain"
        # python3-pip + venv only. TensorRT arrives as a pip wheel from
        # build_trt_engine.py, and CUDA itself comes from the Windows driver via
        # /usr/lib/wsl/lib - installing a CUDA toolkit in the distro would only
        # shadow it with a second, wrong libcuda.
        $prep = @(
            "export DEBIAN_FRONTEND=noninteractive",
            "apt-get update -qq",
            "apt-get install -y -qq python3 python3-pip python3-venv curl ca-certificates >/dev/null",
            "python3 -m pip install --break-system-packages -q -U pip huggingface_hub 2>/dev/null || python3 -m pip install -q -U pip huggingface_hub"
        ) -join " && "
        & wsl.exe -d $Distro -u root -- bash -lc "$prep"
        if ($LASTEXITCODE -ne 0) { Warn "apt/pip prep returned $LASTEXITCODE - continuing, the build may still work" }

        Step "Running the Linux matrix inside WSL"
        # The tools are re-fetched INSIDE the distro rather than read across
        # /mnt/c: DrvFs has no executable bit and its inode semantics confuse
        # anything that stats a file twice. A second download costs seconds.
        # Single-quoted: `$HOME` is expanded by bash inside the distro (root's home
        # there), never by PowerShell, whose own $HOME is the Windows profile.
        $wslWork = '$HOME/uc-build'
        $fetch = "mkdir -p $wslWork/_tools && cd $wslWork/_tools && for f in " +
                 ($TOOL_FILES -join " ") +
                 "; do curl -fsSL -o `$f `"$($TOOLS_URL.TrimEnd('/'))/`$f`" || echo `"  [!] optional/absent: `$f`"; done"
        & wsl.exe -d $Distro -u root -- bash -lc "$fetch"

        $run = "cd $wslWork/_tools && HF_TOKEN='$HF_TOKEN' TOOLS_URL='$TOOLS_URL' " +
               ($(if ($DEST_PREFIX) { "DEST_PREFIX='$DEST_PREFIX' " } else { "" })) +
               "WORKDIR=$wslWork python3 run_matrix.py"
        & wsl.exe -d $Distro -u root -- bash -lc "$run"
        $wslRc = $LASTEXITCODE
        if ($wslRc -eq 0) { Ok "linux_x64 lane complete - every model uploaded" }
        else { Warn ("linux_x64 lane: {0} model(s) FAILED" -f $wslRc) }
    }
}

# -- 5. report -------------------------------------------------------------------------
Head "RESULT  -  $gpu / $SM"
if ($winRc -ne $null) {
    if ($winRc -eq 0) { Write-Host "  win_x64    : OK   (all models)" -ForegroundColor Green }
    else { Write-Host ("  win_x64    : {0} model(s) FAILED" -f $winRc) -ForegroundColor Red }
} else { Write-Host "  win_x64    : not run (-Stage $Stage)" -ForegroundColor DarkGray }

if ($wslSkipReason) {
    # An OS that cannot run WSL2 at all is a FACT about the image, not a failure
    # of this run - Windows Server 2019 (17763) is below the 19041 floor and no
    # amount of retrying changes that. Colour and exit code below treat the two
    # kinds of skip differently, because a caller that branches on the exit code
    # would otherwise see every single Server 2019 build as a failure.
    $expectedSkip = $wslSkipReason -match "19041|WSL2 unavailable"
    $c = if ($expectedSkip) { "DarkGray" } else { "Yellow" }
    Write-Host ("  linux_x64  : SKIPPED{0}" -f $(if ($expectedSkip) { " (expected on this Windows version)" } else { "" })) -ForegroundColor $c
    Write-Host ("               {0}" -f $wslSkipReason) -ForegroundColor $c
    Write-Host "               -> build these cells on a separate Linux GPU VM with bootstrap_linux.sh" -ForegroundColor $c
} elseif ($wslRc -ne $null) {
    if ($wslRc -eq 0) { Write-Host "  linux_x64  : OK   (all models)" -ForegroundColor Green }
    else { Write-Host ("  linux_x64  : {0} model(s) FAILED" -f $wslRc) -ForegroundColor Red }
} else { Write-Host "  linux_x64  : not run (-Stage $Stage)" -ForegroundColor DarkGray }

Write-Host ""
Write-Host "  Verify what actually landed:" -ForegroundColor DarkGray
Write-Host "    https://huggingface.co/faxenoff/code-daemon-embed-v1/tree/main" -ForegroundColor DarkGray
Write-Host ("    every name must end in _{0}.engine for this box" -f $SM) -ForegroundColor DarkGray

$fail = 0
if ($winRc) { $fail += $winRc }
if ($wslRc) { $fail += $wslRc }
# A skip the OS makes unavoidable is not counted. A skip caused by something
# fixable - TCC driver mode, nested virtualization off, an install that failed -
# still is, because those are worth acting on and the operator may only be
# watching the exit code.
if ($wslSkipReason -and -not ($wslSkipReason -match "19041|WSL2 unavailable")) { $fail += 1 }
exit $fail