| param( |
| [int]$Port = 8091, |
| [string]$Python = '' |
| ) |
| $ErrorActionPreference = 'Stop' |
|
|
| $Package = $PSScriptRoot |
| $Meta = Get-Content -LiteralPath (Join-Path $Package 'package.json') -Raw | ConvertFrom-Json |
| $Server = Join-Path $Package 'serve.py' |
| $Model = Join-Path $Package 'model' |
| $Adapter = Join-Path $Package 'adapter' |
| $Template = Join-Path $Model 'chat_template.jinja' |
| $Prompt = Join-Path $Package 'system_prompt.md' |
| $State = Join-Path $Package 'service-state.json' |
| $Stdout = Join-Path $Package 'server.stdout.log' |
| $Stderr = Join-Path $Package 'server.stderr.log' |
| $Offload = Join-Path $Package 'offload' |
| $ReleaseRoot = [IO.Path]::GetFullPath((Join-Path $Package '..')) |
|
|
| if (-not $Python) { $Python = $env:ANRU_PYTHON } |
| if (-not $Python) { |
| $BundledCandidate = [IO.Path]::GetFullPath((Join-Path $Package '..\..\..\work\gemma4-cuda-env\Scripts\python.exe')) |
| if (Test-Path -LiteralPath $BundledCandidate -PathType Leaf) { $Python = $BundledCandidate } |
| } |
| if (-not $Python) { |
| $Python = (Get-Command python -ErrorAction Stop).Source |
| } |
|
|
| foreach ($Path in @($Python, $Server, (Join-Path $Model 'config.json'), (Join-Path $Adapter 'adapter_model.safetensors'), $Template, $Prompt)) { |
| if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw "Missing required file: $Path" } |
| } |
|
|
| $Listener = Get-NetTCPConnection -State Listen -LocalPort $Port -ErrorAction SilentlyContinue | Select-Object -First 1 |
| if ($Listener) { |
| $Owner = Get-CimInstance Win32_Process -Filter "ProcessId=$($Listener.OwningProcess)" |
| $IsSiblingPackage = $Owner.Name -eq 'python.exe' -and |
| $Owner.CommandLine -like "*$ReleaseRoot*serve.py*" |
| if ($Owner.Name -eq 'llama-server.exe' -or $Owner.CommandLine -like "*$Server*" -or $IsSiblingPackage) { |
| Stop-Process -Id $Listener.OwningProcess -Force |
| Wait-Process -Id $Listener.OwningProcess -Timeout 30 -ErrorAction SilentlyContinue |
| Start-Sleep -Seconds 2 |
| } else { |
| throw "Port $Port is owned by $($Owner.Name)" |
| } |
| } |
|
|
| New-Item -ItemType Directory -Path $Offload -Force | Out-Null |
| Remove-Item -LiteralPath $State, $Stdout, $Stderr -Force -ErrorAction SilentlyContinue |
| $env:GEMMA4_MODEL_DIR = $Model |
| $env:GEMMA4_ADAPTER_DIR = $Adapter |
| $env:GEMMA4_TEMPLATE_PATH = $Template |
| $env:GEMMA4_SYSTEM_PROMPT_PATH = $Prompt |
| $env:GEMMA4_STATE_PATH = $State |
| $env:GEMMA4_OFFLOAD_DIR = $Offload |
| $env:GEMMA4_HOST = '127.0.0.1' |
| $env:GEMMA4_PORT = "$Port" |
| $env:GEMMA4_ALIAS = [string]$Meta.alias |
| $env:GEMMA4_LORA_SCALE = '0.45' |
| $env:GEMMA4_MAX_CONTEXT = '131072' |
| $env:GEMMA4_MAX_IMAGE_TOKENS = '1120' |
| $env:GEMMA4_MAX_GPU_MEMORY = [string]$Meta.max_gpu_memory |
| $env:GEMMA4_MAX_CPU_MEMORY = '48GiB' |
| $env:GEMMA4_ATTN_IMPLEMENTATION = 'sdpa' |
|
|
| $Process = Start-Process -FilePath $Python -ArgumentList @('-u', $Server) -WorkingDirectory $Package ` |
| -WindowStyle Hidden -RedirectStandardOutput $Stdout -RedirectStandardError $Stderr -PassThru |
| $Deadline = (Get-Date).AddMinutes(8) |
| $Health = $null |
| do { |
| Start-Sleep -Seconds 2 |
| try { $Health = Invoke-RestMethod "http://127.0.0.1:$Port/health" -TimeoutSec 3 } catch { $Health = $null } |
| if ($Process.HasExited -and $Health.status -ne 'ok') { |
| $Tail = if (Test-Path -LiteralPath $Stderr) { (Get-Content -LiteralPath $Stderr -Tail 120) -join "`n" } else { '' } |
| throw "package service exited`n$Tail" |
| } |
| } until (($Health.status -eq 'ok') -or ((Get-Date) -ge $Deadline)) |
| if ($Health.status -ne 'ok') { throw 'Package service readiness timeout' } |
|
|
| $Live = Get-NetTCPConnection -State Listen -LocalPort $Port | Select-Object -First 1 |
| $Owner = Get-CimInstance Win32_Process -Filter "ProcessId=$($Live.OwningProcess)" |
| if ($Owner.CommandLine -notlike "*$Server*" -or $Health.model -ne $Meta.alias -or |
| [int]$Health.context_tokens -ne 131072 -or [double]$Health.lora_scale -ne 0.45 -or |
| [int]$Health.lora_modules -ne 328 -or [int]$Health.adapter_tensor_count -ne 656 -or |
| -not [bool]$Health.default_system_prompt -or $Health.system_prompt_sha256 -ne $Meta.system_prompt_sha256 -or |
| $Health.attention_implementation -ne 'sdpa') { |
| throw 'Live package validation failed' |
| } |
|
|
| [ordered]@{ |
| state = 'healthy' |
| variant = $Meta.variant |
| alias = $Health.model |
| backend = $Health.backend |
| modalities = @($Health.modalities) |
| context_tokens = [int]$Health.context_tokens |
| attention_implementation = [string]$Health.attention_implementation |
| default_system_prompt = [bool]$Health.default_system_prompt |
| system_prompt_sha256 = [string]$Health.system_prompt_sha256 |
| system_prompt_chars = [int]$Health.system_prompt_chars |
| image_max_soft_tokens = [int]$Health.image_max_soft_tokens |
| lora_scale = [double]$Health.lora_scale |
| lora_modules = [int]$Health.lora_modules |
| adapter_tensor_count = [int]$Health.adapter_tensor_count |
| pid = [int]$Live.OwningProcess |
| port = $Port |
| command = $Owner.CommandLine |
| } | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $State -Encoding UTF8 |
| Get-Content -LiteralPath $State -Raw |
|
|
