| [CmdletBinding()] |
| param( |
| [int]$Port = 8091, |
| [int]$Context = 131072, |
| [switch]$InstallOnly, |
| [switch]$NoBrowser, |
| [switch]$HiddenServer |
| ) |
|
|
| $ErrorActionPreference = 'Stop' |
| $ProgressPreference = 'Continue' |
| $Package = $PSScriptRoot |
| $ModelDir = Join-Path $Package 'llama-model' |
| $BaseModel = Join-Path $ModelDir 'anru-gemma4-12b-base-Q5_K_M.gguf' |
| $LoraModel = Join-Path $ModelDir 'anru-final-lora-Q8_0.gguf' |
| $ChatTemplate = Join-Path $ModelDir 'chat_template.prompt30k.jinja' |
| $UiConfig = Join-Path $ModelDir 'ui-config.json' |
| $RuntimeManifest = Join-Path $Package 'llama-runtime-state.json' |
| $ModelVerification = Join-Path $ModelDir 'model-verification.json' |
|
|
| function Get-LlamaServer { |
| $Candidates = @(Get-ChildItem -LiteralPath $Package -Directory -Filter 'llama-runtime*' -ErrorAction SilentlyContinue | |
| Sort-Object @{ Expression = { if ($_.Name -eq 'llama-runtime') { 0 } else { 1 } } }, Name) |
| foreach ($Directory in $Candidates) { |
| $Found = Get-ChildItem -LiteralPath $Directory.FullName -Recurse -File -Filter 'llama-server.exe' -ErrorAction SilentlyContinue | |
| Select-Object -First 1 |
| if ($Found) { return $Found.FullName } |
| } |
| return $null |
| } |
|
|
| function Install-LlamaRuntime { |
| [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 |
| Write-Host '[bootstrap] llama.cpp runtime is missing; resolving the latest official Windows release...' |
| $Headers = @{ 'User-Agent' = 'Anru-Llama-Bootstrap' } |
| $Release = Invoke-RestMethod -Uri 'https://api.github.com/repos/ggml-org/llama.cpp/releases/latest' ` |
| -Headers $Headers -TimeoutSec 60 |
|
|
| $GpuName = '' |
| $Nvidia = Get-Command 'nvidia-smi.exe' -ErrorAction SilentlyContinue |
| if ($Nvidia) { |
| $GpuName = (& $Nvidia.Source --query-gpu=name --format=csv,noheader 2>$null | Select-Object -First 1).Trim() |
| } |
|
|
| $Assets = @() |
| if ($GpuName) { |
| $CudaFlavor = if ($GpuName -match '(?i)RTX\s*50\d\d|Blackwell') { '13.3' } else { '12.4' } |
| $Main = $Release.assets | Where-Object { $_.name -match "^llama-.*-bin-win-cuda-$([regex]::Escape($CudaFlavor))-x64\.zip$" } | Select-Object -First 1 |
| $Cuda = $Release.assets | Where-Object { $_.name -eq "cudart-llama-bin-win-cuda-$CudaFlavor-x64.zip" } | Select-Object -First 1 |
| if (-not $Main -or -not $Cuda) { throw "No matching CUDA $CudaFlavor Windows assets in release $($Release.tag_name)" } |
| $Assets = @($Main, $Cuda) |
| Write-Host "[bootstrap] detected $GpuName; installing CUDA $CudaFlavor build $($Release.tag_name)." |
| } else { |
| $Main = $Release.assets | Where-Object { $_.name -match '^llama-.*-bin-win-cpu-x64\.zip$' } | Select-Object -First 1 |
| if (-not $Main) { throw "No Windows CPU asset in release $($Release.tag_name)" } |
| $Assets = @($Main) |
| Write-Host "[bootstrap] NVIDIA GPU not detected; installing CPU build $($Release.tag_name)." |
| } |
|
|
| $TempRoot = Join-Path ([IO.Path]::GetTempPath()) ("AnruLlamaBootstrap-" + [guid]::NewGuid().ToString('N')) |
| $RuntimeDir = Join-Path $Package 'llama-runtime' |
| if (Test-Path -LiteralPath $RuntimeDir) { |
| $RuntimeDir = Join-Path $Package ("llama-runtime-" + $Release.tag_name) |
| } |
| New-Item -ItemType Directory -Path $TempRoot, $RuntimeDir -Force | Out-Null |
| try { |
| $Index = 0 |
| $InstalledAssets = @() |
| foreach ($Asset in $Assets) { |
| $Index++ |
| $Zip = Join-Path $TempRoot $Asset.name |
| $Extract = Join-Path $TempRoot ("extract-$Index") |
| Write-Host "[bootstrap] downloading $($Asset.name) ($([math]::Round($Asset.size / 1MB, 1)) MiB)..." |
| Invoke-WebRequest -Uri $Asset.browser_download_url -Headers $Headers -OutFile $Zip -UseBasicParsing -TimeoutSec 1800 |
| if ($Asset.digest -and $Asset.digest -match '^sha256:(.+)$') { |
| $Actual = (Get-FileHash -Algorithm SHA256 -LiteralPath $Zip).Hash |
| if ($Actual -ne $Matches[1].ToUpperInvariant()) { throw "Download checksum mismatch: $($Asset.name)" } |
| } |
| Expand-Archive -LiteralPath $Zip -DestinationPath $Extract -Force |
| foreach ($File in Get-ChildItem -LiteralPath $Extract -Recurse -File) { |
| Copy-Item -LiteralPath $File.FullName -Destination (Join-Path $RuntimeDir $File.Name) -Force |
| } |
| $InstalledAssets += [ordered]@{ name = $Asset.name; bytes = [long]$Asset.size; digest = $Asset.digest } |
| } |
| $Server = Get-ChildItem -LiteralPath $RuntimeDir -Recurse -File -Filter 'llama-server.exe' | Select-Object -First 1 |
| if (-not $Server) { throw 'Downloaded runtime does not contain llama-server.exe' } |
| [ordered]@{ |
| state = 'installed' |
| release = [string]$Release.tag_name |
| gpu = $GpuName |
| installed_at = (Get-Date).ToUniversalTime().ToString('o') |
| runtime = $RuntimeDir |
| server = $Server.FullName |
| assets = $InstalledAssets |
| } | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $RuntimeManifest -Encoding UTF8 |
| Write-Host "[bootstrap] runtime installed at $RuntimeDir" |
| } finally { |
| $ResolvedTemp = [IO.Path]::GetFullPath($TempRoot) |
| $SystemTemp = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()) |
| if ($ResolvedTemp.StartsWith($SystemTemp, [StringComparison]::OrdinalIgnoreCase) -and |
| (Test-Path -LiteralPath $ResolvedTemp)) { |
| Remove-Item -LiteralPath $ResolvedTemp -Recurse -Force -ErrorAction SilentlyContinue |
| } |
| } |
| } |
|
|
| $Server = Get-LlamaServer |
| if (-not $Server) { |
| Install-LlamaRuntime |
| $Server = Get-LlamaServer |
| } |
| if (-not $Server) { throw 'llama-server.exe is still missing after installation' } |
|
|
| if ($InstallOnly) { |
| [ordered]@{ state = 'installed'; server = $Server } | ConvertTo-Json |
| exit 0 |
| } |
|
|
| foreach ($Required in @($BaseModel, $LoraModel, $ChatTemplate, $UiConfig)) { |
| if (-not (Test-Path -LiteralPath $Required -PathType Leaf)) { throw "Missing packaged Llama artifact: $Required" } |
| } |
| if ((Get-Item -LiteralPath $BaseModel).Length -ne 8547249568) { throw 'Packaged Q5 model size mismatch' } |
| if ((Get-Item -LiteralPath $LoraModel).Length -ne 51682496) { throw 'Packaged Q8 LoRA size mismatch' } |
|
|
| $NeedModelHash = $true |
| if (Test-Path -LiteralPath $ModelVerification) { |
| try { |
| $Verified = Get-Content -LiteralPath $ModelVerification -Raw | ConvertFrom-Json |
| $NeedModelHash = -not ($Verified.state -eq 'passed' -and |
| $Verified.base_sha256 -eq '057C364366301EE285E9026ECF417AAE36F4447D5D9CBBC3591447A9FC77A752' -and |
| $Verified.lora_sha256 -eq '3A5060E1CC0C8D4609052EF04A5A60770A64D566E2544B65F6D28F2BA6542037') |
| } catch { $NeedModelHash = $true } |
| } |
| if ($NeedModelHash) { |
| Write-Host '[bootstrap] verifying packaged GGUF model checksums (first launch only)...' |
| $BaseHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $BaseModel).Hash |
| $LoraHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $LoraModel).Hash |
| if ($BaseHash -ne '057C364366301EE285E9026ECF417AAE36F4447D5D9CBBC3591447A9FC77A752') { throw 'Q5 model checksum mismatch' } |
| if ($LoraHash -ne '3A5060E1CC0C8D4609052EF04A5A60770A64D566E2544B65F6D28F2BA6542037') { throw 'Q8 LoRA checksum mismatch' } |
| [ordered]@{ state = 'passed'; base_sha256 = $BaseHash; lora_sha256 = $LoraHash; verified_at = (Get-Date).ToUniversalTime().ToString('o') } | |
| ConvertTo-Json | Set-Content -LiteralPath $ModelVerification -Encoding UTF8 |
| } |
|
|
| $Listener = Get-NetTCPConnection -State Listen -LocalPort $Port -ErrorAction SilentlyContinue | Select-Object -First 1 |
| if ($Listener) { |
| $Owner = Get-CimInstance Win32_Process -Filter "ProcessId=$($Listener.OwningProcess)" |
| $KnownRuntime = $Owner.Name -eq 'llama-server.exe' |
| $KnownNative = $Owner.Name -eq 'python.exe' -and $Owner.CommandLine -like '*anru-gemma4-12b-native-multimodal-release*serve.py*' |
| if ($KnownRuntime -or $KnownNative) { |
| Write-Host "[bootstrap] stopping the previous local model service on port $Port (PID $($Listener.OwningProcess))..." |
| Stop-Process -Id $Listener.OwningProcess -Force |
| Wait-Process -Id $Listener.OwningProcess -Timeout 30 -ErrorAction SilentlyContinue |
| Start-Sleep -Seconds 2 |
| } else { |
| throw "Port $Port is occupied by $($Owner.Name) (PID $($Listener.OwningProcess))" |
| } |
| } |
|
|
| $Alias = 'anru-gemma4-12b-q5-lora045-prompt30k' |
| $RuntimeDir = Split-Path -Parent $Server |
| $BaseForRuntime = '..\llama-model\anru-gemma4-12b-base-Q5_K_M.gguf' |
| $LoraForRuntime = '..\llama-model\anru-final-lora-Q8_0.gguf' |
| $TemplateForRuntime = '..\llama-model\chat_template.prompt30k.jinja' |
| $UiForRuntime = '..\llama-model\ui-config.json' |
| $Arguments = @( |
| '-m', $BaseForRuntime, |
| '--lora-scaled', ('{0}:0.45' -f $LoraForRuntime), |
| '--chat-template-file', $TemplateForRuntime, |
| '--ui-config-file', $UiForRuntime, |
| '-ngl', '99', |
| '-c', "$Context", |
| '-fa', 'on', |
| '-ctk', 'q8_0', |
| '-ctv', 'q8_0', |
| '-b', '1024', |
| '-ub', '256', |
| '-np', '1', |
| '--host', '127.0.0.1', |
| '--port', "$Port", |
| '--metrics', |
| '--jinja', |
| '--webui', |
| '--alias', $Alias |
| ) |
|
|
| $Stdout = Join-Path $Package 'llama-server.stdout.log' |
| $Stderr = Join-Path $Package 'llama-server.stderr.log' |
| Write-Host "[bootstrap] starting llama-server with $Context-token context and LoRA scale 0.45..." |
| if ($HiddenServer) { |
| $Process = Start-Process -FilePath $Server -ArgumentList $Arguments -WorkingDirectory $RuntimeDir ` |
| -WindowStyle Hidden -RedirectStandardOutput $Stdout -RedirectStandardError $Stderr -PassThru |
| } else { |
| $Process = Start-Process -FilePath $Server -ArgumentList $Arguments -WorkingDirectory $RuntimeDir ` |
| -WindowStyle Normal -PassThru |
| } |
|
|
| $Health = $null |
| $Deadline = (Get-Date).AddMinutes(10) |
| do { |
| Start-Sleep -Seconds 2 |
| try { $Health = Invoke-RestMethod -Uri "http://127.0.0.1:$Port/health" -TimeoutSec 3 } catch { $Health = $null } |
| if ($Process.HasExited -and $Health.status -ne 'ok') { |
| $Tail = if (Test-Path -LiteralPath $Stderr) { (Get-Content -LiteralPath $Stderr -Tail 120) -join "`n" } else { '' } |
| throw "llama-server exited during startup (code $($Process.ExitCode))`n$Tail" |
| } |
| } until (($Health.status -eq 'ok') -or ((Get-Date) -ge $Deadline)) |
| if ($Health.status -ne 'ok') { throw 'llama-server readiness timeout' } |
|
|
| $State = [ordered]@{ |
| state = 'healthy' |
| pid = $Process.Id |
| endpoint = "http://127.0.0.1:$Port" |
| model = $Alias |
| context_tokens = $Context |
| lora_scale = 0.45 |
| prompt = 'embedded 30k constitution in chat_template.prompt30k.jinja' |
| prompt_sha256 = '5591AC2C99FDC6C8E21D1D3B87625BB368204548BEC7B63C3FD6598CF23639BF' |
| runtime = $Server |
| started_at = (Get-Date).ToUniversalTime().ToString('o') |
| } |
| $State | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $Package 'llama-service-state.json') -Encoding UTF8 |
| Write-Host "[bootstrap] ready: http://127.0.0.1:$Port" |
| if (-not $NoBrowser) { Start-Process "http://127.0.0.1:$Port" } |
| $State | ConvertTo-Json -Depth 6 |
|
|
