File size: 4,989 Bytes
4e6526a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | param(
[int]$Port = 8091,
[string]$Python = ''
)
$ErrorActionPreference = 'Stop'
$Package = $PSScriptRoot
$Meta = Get-Content -LiteralPath (Join-Path $Package 'package.json') -Raw | ConvertFrom-Json
$Server = Join-Path $Package 'serve.py'
$Model = Join-Path $Package 'model'
$Adapter = Join-Path $Package 'adapter'
$Template = Join-Path $Model 'chat_template.jinja'
$Prompt = Join-Path $Package 'system_prompt.md'
$State = Join-Path $Package 'service-state.json'
$Stdout = Join-Path $Package 'server.stdout.log'
$Stderr = Join-Path $Package 'server.stderr.log'
$Offload = Join-Path $Package 'offload'
$ReleaseRoot = [IO.Path]::GetFullPath((Join-Path $Package '..'))
if (-not $Python) { $Python = $env:ANRU_PYTHON }
if (-not $Python) {
$BundledCandidate = [IO.Path]::GetFullPath((Join-Path $Package '..\..\..\work\gemma4-cuda-env\Scripts\python.exe'))
if (Test-Path -LiteralPath $BundledCandidate -PathType Leaf) { $Python = $BundledCandidate }
}
if (-not $Python) {
$Python = (Get-Command python -ErrorAction Stop).Source
}
foreach ($Path in @($Python, $Server, (Join-Path $Model 'config.json'), (Join-Path $Adapter 'adapter_model.safetensors'), $Template, $Prompt)) {
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw "Missing required file: $Path" }
}
$Listener = Get-NetTCPConnection -State Listen -LocalPort $Port -ErrorAction SilentlyContinue | Select-Object -First 1
if ($Listener) {
$Owner = Get-CimInstance Win32_Process -Filter "ProcessId=$($Listener.OwningProcess)"
$IsSiblingPackage = $Owner.Name -eq 'python.exe' -and
$Owner.CommandLine -like "*$ReleaseRoot*serve.py*"
if ($Owner.Name -eq 'llama-server.exe' -or $Owner.CommandLine -like "*$Server*" -or $IsSiblingPackage) {
Stop-Process -Id $Listener.OwningProcess -Force
Wait-Process -Id $Listener.OwningProcess -Timeout 30 -ErrorAction SilentlyContinue
Start-Sleep -Seconds 2
} else {
throw "Port $Port is owned by $($Owner.Name)"
}
}
New-Item -ItemType Directory -Path $Offload -Force | Out-Null
Remove-Item -LiteralPath $State, $Stdout, $Stderr -Force -ErrorAction SilentlyContinue
$env:GEMMA4_MODEL_DIR = $Model
$env:GEMMA4_ADAPTER_DIR = $Adapter
$env:GEMMA4_TEMPLATE_PATH = $Template
$env:GEMMA4_SYSTEM_PROMPT_PATH = $Prompt
$env:GEMMA4_STATE_PATH = $State
$env:GEMMA4_OFFLOAD_DIR = $Offload
$env:GEMMA4_HOST = '127.0.0.1'
$env:GEMMA4_PORT = "$Port"
$env:GEMMA4_ALIAS = [string]$Meta.alias
$env:GEMMA4_LORA_SCALE = '0.45'
$env:GEMMA4_MAX_CONTEXT = '131072'
$env:GEMMA4_MAX_IMAGE_TOKENS = '1120'
$env:GEMMA4_MAX_GPU_MEMORY = [string]$Meta.max_gpu_memory
$env:GEMMA4_MAX_CPU_MEMORY = '48GiB'
$env:GEMMA4_ATTN_IMPLEMENTATION = 'sdpa'
$Process = Start-Process -FilePath $Python -ArgumentList @('-u', $Server) -WorkingDirectory $Package `
-WindowStyle Hidden -RedirectStandardOutput $Stdout -RedirectStandardError $Stderr -PassThru
$Deadline = (Get-Date).AddMinutes(8)
$Health = $null
do {
Start-Sleep -Seconds 2
try { $Health = Invoke-RestMethod "http://127.0.0.1:$Port/health" -TimeoutSec 3 } catch { $Health = $null }
if ($Process.HasExited -and $Health.status -ne 'ok') {
$Tail = if (Test-Path -LiteralPath $Stderr) { (Get-Content -LiteralPath $Stderr -Tail 120) -join "`n" } else { '' }
throw "package service exited`n$Tail"
}
} until (($Health.status -eq 'ok') -or ((Get-Date) -ge $Deadline))
if ($Health.status -ne 'ok') { throw 'Package service readiness timeout' }
$Live = Get-NetTCPConnection -State Listen -LocalPort $Port | Select-Object -First 1
$Owner = Get-CimInstance Win32_Process -Filter "ProcessId=$($Live.OwningProcess)"
if ($Owner.CommandLine -notlike "*$Server*" -or $Health.model -ne $Meta.alias -or
[int]$Health.context_tokens -ne 131072 -or [double]$Health.lora_scale -ne 0.45 -or
[int]$Health.lora_modules -ne 328 -or [int]$Health.adapter_tensor_count -ne 656 -or
-not [bool]$Health.default_system_prompt -or $Health.system_prompt_sha256 -ne $Meta.system_prompt_sha256 -or
$Health.attention_implementation -ne 'sdpa') {
throw 'Live package validation failed'
}
[ordered]@{
state = 'healthy'
variant = $Meta.variant
alias = $Health.model
backend = $Health.backend
modalities = @($Health.modalities)
context_tokens = [int]$Health.context_tokens
attention_implementation = [string]$Health.attention_implementation
default_system_prompt = [bool]$Health.default_system_prompt
system_prompt_sha256 = [string]$Health.system_prompt_sha256
system_prompt_chars = [int]$Health.system_prompt_chars
image_max_soft_tokens = [int]$Health.image_max_soft_tokens
lora_scale = [double]$Health.lora_scale
lora_modules = [int]$Health.lora_modules
adapter_tensor_count = [int]$Health.adapter_tensor_count
pid = [int]$Live.OwningProcess
port = $Port
command = $Owner.CommandLine
} | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $State -Encoding UTF8
Get-Content -LiteralPath $State -Raw
|