| |
| |
| |
| """Self-contained native API diagnostics; no external datasets or dependencies.""" |
| import argparse |
| import json |
| import time |
| import urllib.request |
|
|
| MODEL = 'DevelopingDad/Gemma4-12B-it-Fast' |
|
|
|
|
| def post(base, payload): |
| request = urllib.request.Request(base.rstrip('/') + '/chat/completions', |
| data=json.dumps(payload).encode(), headers={'Content-Type':'application/json'}) |
| return urllib.request.urlopen(request, timeout=180) |
|
|
|
|
| def request(messages, **kwargs): |
| result = {'model': MODEL, 'messages': messages, 'temperature':0, 'seed':91027, |
| 'max_tokens':256, 'chat_template_kwargs':{'enable_thinking':False}} |
| result.update(kwargs) |
| return result |
|
|
|
|
| def evaluate(base): |
| results = {'schema':'gemma4-fast-generic-api-screen/v1', |
| 'hardware':'one NVIDIA DGX Spark GB10', 'runtime':'vLLM 0.31.0, weight-only target, MTP8, FP8 KV', |
| 'scope':'bounded native API diagnostic; not a standardized quality benchmark', |
| 'sampling':{'temperature':0,'seed':91027}, 'checks':[], 'generation':[]} |
| prompts = [ |
| ('explanation', 'Explain how condensation forms on a cold glass. Include a concrete example and explain the role of temperature.'), |
| ('comparison', 'Compare a binary search tree with a hash table. Explain lookup, insertion, ordering and a suitable use case for each.'), |
| ('code', 'Implement a Python function that merges two sorted integer lists. Include type hints, a docstring, and explain the time and space complexity.'), |
| ] |
| for name, prompt in prompts: |
| body = request([{'role':'user','content':prompt}], max_tokens=512, stream=True, |
| stream_options={'include_usage':True}, cache_salt='generic-public-screen-'+str(time.time_ns())) |
| start = time.monotonic(); first = None; usage = {}; finish = None; chunks = []; reasoning = [] |
| with post(base, body) as response: |
| for line in response: |
| if not line.startswith(b'data: '): continue |
| raw = line[6:].strip() |
| if raw == b'[DONE]': break |
| item = json.loads(raw) |
| if item.get('usage'): usage = item['usage'] |
| for choice in item.get('choices', []): |
| delta = choice.get('delta', {}) |
| if delta.get('content'): |
| if first is None: first = time.monotonic() |
| chunks.append(delta['content']) |
| if delta.get('reasoning_content'): reasoning.append(delta['reasoning_content']) |
| if choice.get('finish_reason'): finish = choice['finish_reason'] |
| end = time.monotonic(); tokens = usage.get('completion_tokens',0) |
| |
| |
| results['generation'].append({'case':name,'max_tokens':512,'finish_reason':finish, |
| 'prompt_tokens':usage.get('prompt_tokens'), 'completion_tokens':tokens, |
| 'first_content_s':None if first is None else round(first-start,4), |
| 'elapsed_s':round(end-start,4), |
| 'approximate_completion_tokens_per_s_after_first_content':None if first is None else round(tokens/max(end-first,1e-9),3), |
| 'nonempty_content':bool(''.join(chunks).strip()),'unexpected_reasoning':bool(reasoning)}) |
|
|
| |
| body = request([{'role':'user','content':'Calculate 137 * 24 - 89. Give the final integer.'}], |
| max_tokens=4096, thinking_token_budget=2048, |
| chat_template_kwargs={'enable_thinking':True}) |
| start = time.monotonic() |
| with post(base,body) as response: item=json.load(response) |
| answer=item['choices'][0]['message'].get('content') or '' |
| results['checks'].append({'case':'reasoning_arithmetic','passed':'3199' in answer.replace(',',''), |
| 'finish_reason':item['choices'][0]['finish_reason'],'elapsed_s':round(time.monotonic()-start,4), |
| 'completion_tokens':item.get('usage',{}).get('completion_tokens'), |
| 'reasoning_tokens':(item.get('usage',{}).get('completion_tokens_details') or {}).get('reasoning_tokens')}) |
|
|
| schema={'type':'object','properties':{'color':{'type':'string','enum':['blue']},'count':{'type':'integer','enum':[3]}},'required':['color','count'],'additionalProperties':False} |
| body=request([{'role':'user','content':'Return color blue and count 3 using the supplied JSON schema.'}], |
| response_format={'type':'json_schema','json_schema':{'name':'sample','strict':True,'schema':schema}}) |
| with post(base,body) as response: item=json.load(response) |
| try: passed=json.loads(item['choices'][0]['message']['content'])=={'color':'blue','count':3} |
| except (ValueError,TypeError,KeyError): passed=False |
| results['checks'].append({'case':'strict_json_schema','passed':passed,'finish_reason':item['choices'][0]['finish_reason']}) |
|
|
| tool={'type':'function','function':{'name':'get_temperature','description':'Read a temperature for a named city.', |
| 'parameters':{'type':'object','properties':{'city':{'type':'string'}},'required':['city'],'additionalProperties':False}}} |
| messages=[{'role':'user','content':'Use the temperature tool for Paris.'}] |
| body=request(messages,tools=[tool],tool_choice={'type':'function','function':{'name':'get_temperature'}}) |
| with post(base,body) as response: item=json.load(response) |
| message=item['choices'][0]['message']; calls=message.get('tool_calls') or [] |
| passed=bool(calls) and calls[0]['function']['name']=='get_temperature' and json.loads(calls[0]['function']['arguments']).get('city','').lower()=='paris' |
| results['checks'].append({'case':'tool_call','passed':passed,'finish_reason':item['choices'][0]['finish_reason']}) |
| if passed: |
| messages.extend([{'role':'assistant','content':message.get('content'),'tool_calls':calls}, |
| {'role':'tool','tool_call_id':calls[0]['id'],'content':'{"city":"Paris","temperature_celsius":17}'}]) |
| with post(base,request(messages,tools=[tool],tool_choice='none')) as response: item=json.load(response) |
| answer=item['choices'][0]['message'].get('content') or '' |
| results['checks'].append({'case':'tool_result_continuation','passed':'17' in answer,'finish_reason':item['choices'][0]['finish_reason']}) |
| results['all_checks_passed']=all(x['passed'] for x in results['checks']) and len(results['checks'])==4 |
| results['generation_nonempty']=all(x['nonempty_content'] for x in results['generation']) |
| return results |
|
|
|
|
| def main(): |
| parser=argparse.ArgumentParser(description=__doc__) |
| parser.add_argument('--base-url',default='http://127.0.0.1:8000/v1') |
| parser.add_argument('--output',default='generic-results.json') |
| args=parser.parse_args() |
| results=evaluate(args.base_url) |
| with open(args.output,'w') as handle: json.dump(results,handle,indent=2); handle.write('\n') |
| print(json.dumps(results,indent=2)) |
| return 0 if results['all_checks_passed'] and results['generation_nonempty'] else 1 |
|
|
| if __name__=='__main__': raise SystemExit(main()) |
|
|
