File size: 6,436 Bytes
dbb8e24 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | param([int]$Port = 8091)
$ErrorActionPreference = 'Stop'
$Package = $PSScriptRoot
$Meta = Get-Content -LiteralPath (Join-Path $Package 'package.json') -Raw | ConvertFrom-Json
$Image = Join-Path $Package 'test-whoami.png'
$Health = Invoke-RestMethod "http://127.0.0.1:$Port/health" -TimeoutSec 10
if ($Health.model -ne $Meta.alias) { throw 'The running service is not this package' }
if (-not [bool]$Health.default_system_prompt -or $Health.system_prompt_sha256 -ne $Meta.system_prompt_sha256) {
throw 'The packaged 30k system prompt is not active'
}
$IdentityBody = [ordered]@{
model = $Meta.alias
messages = @(@{ role = 'user'; content = '请只用一句简短中文回答：你是谁？' })
temperature = 0
max_tokens = 64
} | ConvertTo-Json -Depth 8
$Identity = Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$Port/v1/chat/completions" `
-ContentType 'application/json; charset=utf-8' -Body ([Text.Encoding]::UTF8.GetBytes($IdentityBody)) -TimeoutSec 600
$LiteralIdentity = [string]$Identity.choices[0].message.content
if ($LiteralIdentity -notmatch '(?i)安若|Anru') { throw "Identity prompt test failed: $LiteralIdentity" }
if (-not [bool]$Identity.system_prompt.default_injected -or [int]$Identity.usage.prompt_tokens -lt 1000) {
throw 'Identity request did not inject the packaged 30k prompt'
}
$TextBody = [ordered]@{
model = $Meta.alias
use_default_system_prompt = $false
messages = @(@{ role = 'user'; content = 'Answer only the number: 2+2=' })
temperature = 0
max_tokens = 12
} | ConvertTo-Json -Depth 8
$Text = Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$Port/v1/chat/completions" `
-ContentType 'application/json; charset=utf-8' -Body ([Text.Encoding]::UTF8.GetBytes($TextBody)) -TimeoutSec 300
if ($Text.choices[0].message.content -notmatch '4') { throw 'Text smoke test failed' }
$ImageBody = [ordered]@{
model = $Meta.alias
use_default_system_prompt = $false
messages = @(@{
role = 'user'
content = @(
@{ type = 'image_url'; image_url = @{ url = $Image } },
@{ type = 'text'; text = 'Transcribe the entire output line immediately below whoami. Include the text before and after the backslash. Output that one line only.' }
)
})
temperature = 0
max_tokens = 20
} | ConvertTo-Json -Depth 10
$ImageResult = Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$Port/v1/chat/completions" `
-ContentType 'application/json; charset=utf-8' -Body ([Text.Encoding]::UTF8.GetBytes($ImageBody)) -TimeoutSec 300
$LiteralImage = [string]$ImageResult.choices[0].message.content
if ($LiteralImage -notmatch '(?i)desktop-nusvlb5\\merlin') { throw "Image smoke test failed: $LiteralImage" }
$AudioPath = Join-Path $Package 'test-audio.wav'
$AudioBody = [ordered]@{
model = $Meta.alias
use_default_system_prompt = $false
messages = @(@{
role = 'user'
content = @(
@{ type = 'audio'; audio = $AudioPath },
@{ type = 'text'; text = 'Write the three digits spoken in this audio, in order. Answer digits only.' }
)
})
temperature = 0
max_tokens = 12
} | ConvertTo-Json -Depth 10
$AudioResult = Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$Port/v1/chat/completions" `
-ContentType 'application/json; charset=utf-8' -Body ([Text.Encoding]::UTF8.GetBytes($AudioBody)) -TimeoutSec 300
$LiteralAudio = [string]$AudioResult.choices[0].message.content
if ($LiteralAudio -notmatch '123') { throw "Audio smoke test failed: $LiteralAudio" }
$VideoPath = Join-Path $Package 'test-video.mp4'
$VideoBody = [ordered]@{
model = $Meta.alias
use_default_system_prompt = $false
messages = @(@{
role = 'user'
content = @(
@{ type = 'video'; video = $VideoPath },
@{ type = 'text'; text = 'What three-digit VIDEO CODE is displayed throughout this video? Answer digits only.' }
)
})
temperature = 0
max_tokens = 12
} | ConvertTo-Json -Depth 10
$VideoResult = Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$Port/v1/chat/completions" `
-ContentType 'application/json; charset=utf-8' -Body ([Text.Encoding]::UTF8.GetBytes($VideoBody)) -TimeoutSec 300
$LiteralVideo = [string]$VideoResult.choices[0].message.content
if ($LiteralVideo -notmatch '731') { throw "Video smoke test failed: $LiteralVideo" }
$Record = [ordered]@{
state = 'passed'
variant = $Meta.variant
alias = $Meta.alias
backend = $Health.backend
context_tokens = [int]$Health.context_tokens
lora_scale = [double]$Health.lora_scale
system_prompt = [ordered]@{
file = [string]$Health.system_prompt_file
sha256 = [string]$Health.system_prompt_sha256
bytes = [int]$Health.system_prompt_bytes
chars = [int]$Health.system_prompt_chars
lines = [int]$Health.system_prompt_lines
identity_literal_output = $LiteralIdentity
prompt_tokens = [int]$Identity.usage.prompt_tokens
completion_tokens = [int]$Identity.usage.completion_tokens
seconds = [double]$Identity.timing.elapsed_seconds
}
text = [ordered]@{
literal_output = [string]$Text.choices[0].message.content
prompt_tokens = [int]$Text.usage.prompt_tokens
completion_tokens = [int]$Text.usage.completion_tokens
seconds = [double]$Text.timing.elapsed_seconds
}
image = [ordered]@{
input = $Image
literal_output = $LiteralImage
prompt_tokens = [int]$ImageResult.usage.prompt_tokens
completion_tokens = [int]$ImageResult.usage.completion_tokens
seconds = [double]$ImageResult.timing.elapsed_seconds
}
audio = [ordered]@{
input = $AudioPath
literal_output = $LiteralAudio
prompt_tokens = [int]$AudioResult.usage.prompt_tokens
completion_tokens = [int]$AudioResult.usage.completion_tokens
seconds = [double]$AudioResult.timing.elapsed_seconds
}
video = [ordered]@{
input = $VideoPath
literal_output = $LiteralVideo
prompt_tokens = [int]$VideoResult.usage.prompt_tokens
completion_tokens = [int]$VideoResult.usage.completion_tokens
seconds = [double]$VideoResult.timing.elapsed_seconds
}
}
$Record | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $Package 'smoke-test-result.json') -Encoding UTF8
$Record | ConvertTo-Json -Depth 8
|