File size: 7,248 Bytes
231d9a6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | #!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 DevelopingDad
"""Self-contained native API diagnostics; no external datasets or dependencies."""
import argparse
import json
import time
import urllib.request
MODEL = 'DevelopingDad/Gemma4-12B-it-Fast'
def post(base, payload):
request = urllib.request.Request(base.rstrip('/') + '/chat/completions',
data=json.dumps(payload).encode(), headers={'Content-Type':'application/json'})
return urllib.request.urlopen(request, timeout=180)
def request(messages, **kwargs):
result = {'model': MODEL, 'messages': messages, 'temperature':0, 'seed':91027,
'max_tokens':256, 'chat_template_kwargs':{'enable_thinking':False}}
result.update(kwargs)
return result
def evaluate(base):
results = {'schema':'gemma4-fast-generic-api-screen/v1',
'hardware':'one NVIDIA DGX Spark GB10', 'runtime':'vLLM 0.31.0, weight-only target, MTP8, FP8 KV',
'scope':'bounded native API diagnostic; not a standardized quality benchmark',
'sampling':{'temperature':0,'seed':91027}, 'checks':[], 'generation':[]}
prompts = [
('explanation', 'Explain how condensation forms on a cold glass. Include a concrete example and explain the role of temperature.'),
('comparison', 'Compare a binary search tree with a hash table. Explain lookup, insertion, ordering and a suitable use case for each.'),
('code', 'Implement a Python function that merges two sorted integer lists. Include type hints, a docstring, and explain the time and space complexity.'),
]
for name, prompt in prompts:
body = request([{'role':'user','content':prompt}], max_tokens=512, stream=True,
stream_options={'include_usage':True}, cache_salt='generic-public-screen-'+str(time.time_ns()))
start = time.monotonic(); first = None; usage = {}; finish = None; chunks = []; reasoning = []
with post(base, body) as response:
for line in response:
if not line.startswith(b'data: '): continue
raw = line[6:].strip()
if raw == b'[DONE]': break
item = json.loads(raw)
if item.get('usage'): usage = item['usage']
for choice in item.get('choices', []):
delta = choice.get('delta', {})
if delta.get('content'):
if first is None: first = time.monotonic()
chunks.append(delta['content'])
if delta.get('reasoning_content'): reasoning.append(delta['reasoning_content'])
if choice.get('finish_reason'): finish = choice['finish_reason']
end = time.monotonic(); tokens = usage.get('completion_tokens',0)
# Completion usage can include final control tokens; this is an explicitly
# approximate decode rate, not the old conservative delivery metric.
results['generation'].append({'case':name,'max_tokens':512,'finish_reason':finish,
'prompt_tokens':usage.get('prompt_tokens'), 'completion_tokens':tokens,
'first_content_s':None if first is None else round(first-start,4),
'elapsed_s':round(end-start,4),
'approximate_completion_tokens_per_s_after_first_content':None if first is None else round(tokens/max(end-first,1e-9),3),
'nonempty_content':bool(''.join(chunks).strip()),'unexpected_reasoning':bool(reasoning)})
# Explicit reasoning and a completion allowance exceeding its reasoning cap.
body = request([{'role':'user','content':'Calculate 137 * 24 - 89. Give the final integer.'}],
max_tokens=4096, thinking_token_budget=2048,
chat_template_kwargs={'enable_thinking':True})
start = time.monotonic()
with post(base,body) as response: item=json.load(response)
answer=item['choices'][0]['message'].get('content') or ''
results['checks'].append({'case':'reasoning_arithmetic','passed':'3199' in answer.replace(',',''),
'finish_reason':item['choices'][0]['finish_reason'],'elapsed_s':round(time.monotonic()-start,4),
'completion_tokens':item.get('usage',{}).get('completion_tokens'),
'reasoning_tokens':(item.get('usage',{}).get('completion_tokens_details') or {}).get('reasoning_tokens')})
schema={'type':'object','properties':{'color':{'type':'string','enum':['blue']},'count':{'type':'integer','enum':[3]}},'required':['color','count'],'additionalProperties':False}
body=request([{'role':'user','content':'Return color blue and count 3 using the supplied JSON schema.'}],
response_format={'type':'json_schema','json_schema':{'name':'sample','strict':True,'schema':schema}})
with post(base,body) as response: item=json.load(response)
try: passed=json.loads(item['choices'][0]['message']['content'])=={'color':'blue','count':3}
except (ValueError,TypeError,KeyError): passed=False
results['checks'].append({'case':'strict_json_schema','passed':passed,'finish_reason':item['choices'][0]['finish_reason']})
tool={'type':'function','function':{'name':'get_temperature','description':'Read a temperature for a named city.',
'parameters':{'type':'object','properties':{'city':{'type':'string'}},'required':['city'],'additionalProperties':False}}}
messages=[{'role':'user','content':'Use the temperature tool for Paris.'}]
body=request(messages,tools=[tool],tool_choice={'type':'function','function':{'name':'get_temperature'}})
with post(base,body) as response: item=json.load(response)
message=item['choices'][0]['message']; calls=message.get('tool_calls') or []
passed=bool(calls) and calls[0]['function']['name']=='get_temperature' and json.loads(calls[0]['function']['arguments']).get('city','').lower()=='paris'
results['checks'].append({'case':'tool_call','passed':passed,'finish_reason':item['choices'][0]['finish_reason']})
if passed:
messages.extend([{'role':'assistant','content':message.get('content'),'tool_calls':calls},
{'role':'tool','tool_call_id':calls[0]['id'],'content':'{"city":"Paris","temperature_celsius":17}'}])
with post(base,request(messages,tools=[tool],tool_choice='none')) as response: item=json.load(response)
answer=item['choices'][0]['message'].get('content') or ''
results['checks'].append({'case':'tool_result_continuation','passed':'17' in answer,'finish_reason':item['choices'][0]['finish_reason']})
results['all_checks_passed']=all(x['passed'] for x in results['checks']) and len(results['checks'])==4
results['generation_nonempty']=all(x['nonempty_content'] for x in results['generation'])
return results
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url',default='http://127.0.0.1:8000/v1')
parser.add_argument('--output',default='generic-results.json')
args=parser.parse_args()
results=evaluate(args.base_url)
with open(args.output,'w') as handle: json.dump(results,handle,indent=2); handle.write('\n')
print(json.dumps(results,indent=2))
return 0 if results['all_checks_passed'] and results['generation_nonempty'] else 1
if __name__=='__main__': raise SystemExit(main())
|