[CmdletBinding()] param( [int]$Port = 8091, [int]$Context = 131072, [switch]$InstallOnly, [switch]$NoBrowser, [switch]$HiddenServer ) $ErrorActionPreference = 'Stop' $ProgressPreference = 'Continue' $Package = $PSScriptRoot $ModelDir = Join-Path $Package 'llama-model' $BaseModel = Join-Path $ModelDir 'anru-gemma4-12b-base-Q5_K_M.gguf' $LoraModel = Join-Path $ModelDir 'anru-final-lora-Q8_0.gguf' $ChatTemplate = Join-Path $ModelDir 'chat_template.prompt30k.jinja' $UiConfig = Join-Path $ModelDir 'ui-config.json' $RuntimeManifest = Join-Path $Package 'llama-runtime-state.json' $ModelVerification = Join-Path $ModelDir 'model-verification.json' function Get-LlamaServer { $Candidates = @(Get-ChildItem -LiteralPath $Package -Directory -Filter 'llama-runtime*' -ErrorAction SilentlyContinue | Sort-Object @{ Expression = { if ($_.Name -eq 'llama-runtime') { 0 } else { 1 } } }, Name) foreach ($Directory in $Candidates) { $Found = Get-ChildItem -LiteralPath $Directory.FullName -Recurse -File -Filter 'llama-server.exe' -ErrorAction SilentlyContinue | Select-Object -First 1 if ($Found) { return $Found.FullName } } return $null } function Install-LlamaRuntime { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 Write-Host '[bootstrap] llama.cpp runtime is missing; resolving the latest official Windows release...' $Headers = @{ 'User-Agent' = 'Anru-Llama-Bootstrap' } $Release = Invoke-RestMethod -Uri 'https://api.github.com/repos/ggml-org/llama.cpp/releases/latest' ` -Headers $Headers -TimeoutSec 60 $GpuName = '' $Nvidia = Get-Command 'nvidia-smi.exe' -ErrorAction SilentlyContinue if ($Nvidia) { $GpuName = (& $Nvidia.Source --query-gpu=name --format=csv,noheader 2>$null | Select-Object -First 1).Trim() } $Assets = @() if ($GpuName) { $CudaFlavor = if ($GpuName -match '(?i)RTX\s*50\d\d|Blackwell') { '13.3' } else { '12.4' } $Main = $Release.assets | Where-Object { $_.name -match "^llama-.*-bin-win-cuda-$([regex]::Escape($CudaFlavor))-x64\.zip$" } | Select-Object -First 1 $Cuda = $Release.assets | Where-Object { $_.name -eq "cudart-llama-bin-win-cuda-$CudaFlavor-x64.zip" } | Select-Object -First 1 if (-not $Main -or -not $Cuda) { throw "No matching CUDA $CudaFlavor Windows assets in release $($Release.tag_name)" } $Assets = @($Main, $Cuda) Write-Host "[bootstrap] detected $GpuName; installing CUDA $CudaFlavor build $($Release.tag_name)." } else { $Main = $Release.assets | Where-Object { $_.name -match '^llama-.*-bin-win-cpu-x64\.zip$' } | Select-Object -First 1 if (-not $Main) { throw "No Windows CPU asset in release $($Release.tag_name)" } $Assets = @($Main) Write-Host "[bootstrap] NVIDIA GPU not detected; installing CPU build $($Release.tag_name)." } $TempRoot = Join-Path ([IO.Path]::GetTempPath()) ("AnruLlamaBootstrap-" + [guid]::NewGuid().ToString('N')) $RuntimeDir = Join-Path $Package 'llama-runtime' if (Test-Path -LiteralPath $RuntimeDir) { $RuntimeDir = Join-Path $Package ("llama-runtime-" + $Release.tag_name) } New-Item -ItemType Directory -Path $TempRoot, $RuntimeDir -Force | Out-Null try { $Index = 0 $InstalledAssets = @() foreach ($Asset in $Assets) { $Index++ $Zip = Join-Path $TempRoot $Asset.name $Extract = Join-Path $TempRoot ("extract-$Index") Write-Host "[bootstrap] downloading $($Asset.name) ($([math]::Round($Asset.size / 1MB, 1)) MiB)..." Invoke-WebRequest -Uri $Asset.browser_download_url -Headers $Headers -OutFile $Zip -UseBasicParsing -TimeoutSec 1800 if ($Asset.digest -and $Asset.digest -match '^sha256:(.+)$') { $Actual = (Get-FileHash -Algorithm SHA256 -LiteralPath $Zip).Hash if ($Actual -ne $Matches[1].ToUpperInvariant()) { throw "Download checksum mismatch: $($Asset.name)" } } Expand-Archive -LiteralPath $Zip -DestinationPath $Extract -Force foreach ($File in Get-ChildItem -LiteralPath $Extract -Recurse -File) { Copy-Item -LiteralPath $File.FullName -Destination (Join-Path $RuntimeDir $File.Name) -Force } $InstalledAssets += [ordered]@{ name = $Asset.name; bytes = [long]$Asset.size; digest = $Asset.digest } } $Server = Get-ChildItem -LiteralPath $RuntimeDir -Recurse -File -Filter 'llama-server.exe' | Select-Object -First 1 if (-not $Server) { throw 'Downloaded runtime does not contain llama-server.exe' } [ordered]@{ state = 'installed' release = [string]$Release.tag_name gpu = $GpuName installed_at = (Get-Date).ToUniversalTime().ToString('o') runtime = $RuntimeDir server = $Server.FullName assets = $InstalledAssets } | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $RuntimeManifest -Encoding UTF8 Write-Host "[bootstrap] runtime installed at $RuntimeDir" } finally { $ResolvedTemp = [IO.Path]::GetFullPath($TempRoot) $SystemTemp = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()) if ($ResolvedTemp.StartsWith($SystemTemp, [StringComparison]::OrdinalIgnoreCase) -and (Test-Path -LiteralPath $ResolvedTemp)) { Remove-Item -LiteralPath $ResolvedTemp -Recurse -Force -ErrorAction SilentlyContinue } } } $Server = Get-LlamaServer if (-not $Server) { Install-LlamaRuntime $Server = Get-LlamaServer } if (-not $Server) { throw 'llama-server.exe is still missing after installation' } if ($InstallOnly) { [ordered]@{ state = 'installed'; server = $Server } | ConvertTo-Json exit 0 } foreach ($Required in @($BaseModel, $LoraModel, $ChatTemplate, $UiConfig)) { if (-not (Test-Path -LiteralPath $Required -PathType Leaf)) { throw "Missing packaged Llama artifact: $Required" } } if ((Get-Item -LiteralPath $BaseModel).Length -ne 8547249568) { throw 'Packaged Q5 model size mismatch' } if ((Get-Item -LiteralPath $LoraModel).Length -ne 51682496) { throw 'Packaged Q8 LoRA size mismatch' } $NeedModelHash = $true if (Test-Path -LiteralPath $ModelVerification) { try { $Verified = Get-Content -LiteralPath $ModelVerification -Raw | ConvertFrom-Json $NeedModelHash = -not ($Verified.state -eq 'passed' -and $Verified.base_sha256 -eq '057C364366301EE285E9026ECF417AAE36F4447D5D9CBBC3591447A9FC77A752' -and $Verified.lora_sha256 -eq '3A5060E1CC0C8D4609052EF04A5A60770A64D566E2544B65F6D28F2BA6542037') } catch { $NeedModelHash = $true } } if ($NeedModelHash) { Write-Host '[bootstrap] verifying packaged GGUF model checksums (first launch only)...' $BaseHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $BaseModel).Hash $LoraHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $LoraModel).Hash if ($BaseHash -ne '057C364366301EE285E9026ECF417AAE36F4447D5D9CBBC3591447A9FC77A752') { throw 'Q5 model checksum mismatch' } if ($LoraHash -ne '3A5060E1CC0C8D4609052EF04A5A60770A64D566E2544B65F6D28F2BA6542037') { throw 'Q8 LoRA checksum mismatch' } [ordered]@{ state = 'passed'; base_sha256 = $BaseHash; lora_sha256 = $LoraHash; verified_at = (Get-Date).ToUniversalTime().ToString('o') } | ConvertTo-Json | Set-Content -LiteralPath $ModelVerification -Encoding UTF8 } $Listener = Get-NetTCPConnection -State Listen -LocalPort $Port -ErrorAction SilentlyContinue | Select-Object -First 1 if ($Listener) { $Owner = Get-CimInstance Win32_Process -Filter "ProcessId=$($Listener.OwningProcess)" $KnownRuntime = $Owner.Name -eq 'llama-server.exe' $KnownNative = $Owner.Name -eq 'python.exe' -and $Owner.CommandLine -like '*anru-gemma4-12b-native-multimodal-release*serve.py*' if ($KnownRuntime -or $KnownNative) { Write-Host "[bootstrap] stopping the previous local model service on port $Port (PID $($Listener.OwningProcess))..." Stop-Process -Id $Listener.OwningProcess -Force Wait-Process -Id $Listener.OwningProcess -Timeout 30 -ErrorAction SilentlyContinue Start-Sleep -Seconds 2 } else { throw "Port $Port is occupied by $($Owner.Name) (PID $($Listener.OwningProcess))" } } $Alias = 'anru-gemma4-12b-q5-lora045-prompt30k' $RuntimeDir = Split-Path -Parent $Server $BaseForRuntime = '..\llama-model\anru-gemma4-12b-base-Q5_K_M.gguf' $LoraForRuntime = '..\llama-model\anru-final-lora-Q8_0.gguf' $TemplateForRuntime = '..\llama-model\chat_template.prompt30k.jinja' $UiForRuntime = '..\llama-model\ui-config.json' $Arguments = @( '-m', $BaseForRuntime, '--lora-scaled', ('{0}:0.45' -f $LoraForRuntime), '--chat-template-file', $TemplateForRuntime, '--ui-config-file', $UiForRuntime, '-ngl', '99', '-c', "$Context", '-fa', 'on', '-ctk', 'q8_0', '-ctv', 'q8_0', '-b', '1024', '-ub', '256', '-np', '1', '--host', '127.0.0.1', '--port', "$Port", '--metrics', '--jinja', '--webui', '--alias', $Alias ) $Stdout = Join-Path $Package 'llama-server.stdout.log' $Stderr = Join-Path $Package 'llama-server.stderr.log' Write-Host "[bootstrap] starting llama-server with $Context-token context and LoRA scale 0.45..." if ($HiddenServer) { $Process = Start-Process -FilePath $Server -ArgumentList $Arguments -WorkingDirectory $RuntimeDir ` -WindowStyle Hidden -RedirectStandardOutput $Stdout -RedirectStandardError $Stderr -PassThru } else { $Process = Start-Process -FilePath $Server -ArgumentList $Arguments -WorkingDirectory $RuntimeDir ` -WindowStyle Normal -PassThru } $Health = $null $Deadline = (Get-Date).AddMinutes(10) do { Start-Sleep -Seconds 2 try { $Health = Invoke-RestMethod -Uri "http://127.0.0.1:$Port/health" -TimeoutSec 3 } catch { $Health = $null } if ($Process.HasExited -and $Health.status -ne 'ok') { $Tail = if (Test-Path -LiteralPath $Stderr) { (Get-Content -LiteralPath $Stderr -Tail 120) -join "`n" } else { '' } throw "llama-server exited during startup (code $($Process.ExitCode))`n$Tail" } } until (($Health.status -eq 'ok') -or ((Get-Date) -ge $Deadline)) if ($Health.status -ne 'ok') { throw 'llama-server readiness timeout' } $State = [ordered]@{ state = 'healthy' pid = $Process.Id endpoint = "http://127.0.0.1:$Port" model = $Alias context_tokens = $Context lora_scale = 0.45 prompt = 'embedded 30k constitution in chat_template.prompt30k.jinja' prompt_sha256 = '5591AC2C99FDC6C8E21D1D3B87625BB368204548BEC7B63C3FD6598CF23639BF' runtime = $Server started_at = (Get-Date).ToUniversalTime().ToString('o') } $State | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $Package 'llama-service-state.json') -Encoding UTF8 Write-Host "[bootstrap] ready: http://127.0.0.1:$Port" if (-not $NoBrowser) { Start-Process "http://127.0.0.1:$Port" } $State | ConvertTo-Json -Depth 6