environment/training-source/source-snapshot/tools/run_decoder_matrix.ps1

param(
    [Parameter(Mandatory = $true)]
    [string]$Manifest,
    [Parameter(Mandatory = $true)]
    [string]$Model,
    [Parameter(Mandatory = $true)]
    [string]$OutputRoot,
    [string[]]$Seeds = @('0', '1', '2'),
    [int]$Steps = 12000,
    [int]$Images = 2500,
    [int]$Layers = 3,
    [int]$Size = 224,
    [string]$EvaluationSplit = "validation"
)

$ErrorActionPreference = "Stop"
$repo = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
$manifestPath = (Resolve-Path $Manifest).Path
$modelPath = (Resolve-Path $Model).Path
$manifestHash = (Get-FileHash -LiteralPath $manifestPath -Algorithm SHA256).Hash.ToLowerInvariant()
$modelHash = (Get-FileHash -LiteralPath $modelPath -Algorithm SHA256).Hash.ToLowerInvariant()
$outputPath = [System.IO.Path]::GetFullPath($OutputRoot)
New-Item -ItemType Directory -Force -Path $outputPath | Out-Null
$cargo = (Get-Command cargo).Source
$seedValues = @(
    $Seeds | ForEach-Object { $_ -split ',' } | ForEach-Object {
        $value = 0L
        if (-not [long]::TryParse($_.Trim(), [ref]$value) -or $value -lt 0) {
            throw "invalid seed: $_"
        }
        $value
    }
)
if ($seedValues.Count -eq 0 -or @($seedValues | Select-Object -Unique).Count -ne $seedValues.Count) {
    throw 'provide at least one seed, without duplicates'
}

function Run-Cargo(
    [string[]]$Arguments,
    [string]$Stdout,
    [string]$Stderr,
    [string]$LogLevel
) {
    $env:RUST_LOG = $LogLevel
    Push-Location $repo
    $oldErrorPreference = $ErrorActionPreference
    try {
        # Windows PowerShell may wrap native stderr as ErrorRecord objects;
        # Continue preserves the native exit code while the two streams are
        # retained independently.
        $ErrorActionPreference = "Continue"
        & $cargo @Arguments 1> $Stdout 2> $Stderr
        $exitCode = $LASTEXITCODE
    } finally {
        $ErrorActionPreference = $oldErrorPreference
        Pop-Location
    }
    if ($exitCode -ne 0) {
        throw "cargo exited with code $exitCode; see $Stdout and $Stderr"
    }
}

foreach ($seed in $seedValues) {
    $runDir = Join-Path $outputPath "${Layers}l${Size}-seed${seed}"
    New-Item -ItemType Directory -Force -Path $runDir | Out-Null
    $decoder = Join-Path $runDir "decoder.bin"
    $training = Join-Path $runDir "training.json"
    $quality = Join-Path $runDir "quality-imagenette-$EvaluationSplit.json"

    if (-not ((Test-Path -LiteralPath $training) -and (Test-Path -LiteralPath $decoder))) {
        Write-Output "training seed $seed at $(Get-Date -Format o)"
        Run-Cargo `
            -Arguments @(
                "run", "--release", "--example", "train_decoder", "--",
                $manifestPath, $modelPath, "$Steps", "$Images", "$Layers", "$Size", "$seed", $runDir
            ) `
            -Stdout (Join-Path $runDir "training.stdout.log") `
            -Stderr (Join-Path $runDir "training.stderr.log") `
            -LogLevel "info"
    } else {
        Write-Output "seed $seed training outputs already exist; preserving them"
    }

    $record = Get-Content -Raw -LiteralPath $training | ConvertFrom-Json
    if (
        $record.seed -ne $seed -or
        $record.steps -ne $Steps -or
        $record.requested_images -ne $Images -or
        $record.encoder_layers -ne $Layers -or
        $record.image_size -ne $Size -or
        $record.dataset_manifest_sha256 -ne $manifestHash -or
        $record.model_sha256 -ne $modelHash
    ) {
        throw "existing training record does not match requested cell: $training"
    }

    if (-not (Test-Path -LiteralPath $quality)) {
        Write-Output "evaluating seed $seed at $(Get-Date -Format o)"
        Run-Cargo `
            -Arguments @(
                "run", "--release", "--example", "evaluate_decoder", "--",
                "--manifest", $manifestPath,
                "--model", $modelPath,
                "--decoder", $decoder,
                "--split", $EvaluationSplit,
                "--layers", "$Layers",
                "--size", "$Size",
                "--output", $quality,
                "--samples", "12"
            ) `
            -Stdout (Join-Path $runDir "evaluation.stdout.log") `
            -Stderr (Join-Path $runDir "evaluation.stderr.log") `
            -LogLevel "warn"
    } else {
        Write-Output "seed $seed quality output already exists; preserving it"
    }

    $qualityRecord = Get-Content -Raw -LiteralPath $quality | ConvertFrom-Json
    if (
        $qualityRecord.split -ne $EvaluationSplit -or
        $qualityRecord.encoder_layers -ne $Layers -or
        $qualityRecord.image_size -ne $Size -or
        $qualityRecord.manifest_sha256 -ne $manifestHash -or
        $qualityRecord.model_sha256 -ne $modelHash -or
        $qualityRecord.decoder_sha256 -ne $record.decoder_sha256
    ) {
        throw "quality record does not match its training cell: $quality"
    }

    $safeTensors = Join-Path $runDir "decoder.safetensors"
    if (-not (Test-Path -LiteralPath $safeTensors)) {
        & python (Join-Path $repo "tools\convert_decoder.py") `
            --training $training `
            --input $decoder `
            --output $safeTensors
        if ($LASTEXITCODE -ne 0) {
            throw "decoder SafeTensors conversion failed for seed $seed"
        }
    }
}

Write-Output "decoder matrix complete at $(Get-Date -Format o)"