import json,time,requests,base64,re,subprocess,sys,unicodedata,concurrent.futures,statistics from pathlib import Path from decimal import Decimal R=Path(__file__).parent TOOLS=[{'type':'function','function':{'name':'add','description':'Add two integers','parameters':{'type':'object','properties':{'a':{'type':'integer'},'b':{'type':'integer'}},'required':['a','b']}}}] def request(base,payload): payload=dict(payload,stream=True,stream_options={'include_usage':True});start=time.perf_counter();first=None;text='';usage={};finish=None;calls={} with requests.post(base+'/v1/chat/completions',json=payload,stream=True,timeout=(15,360)) as r: r.raise_for_status() for line in r.iter_lines(): if not line.startswith(b'data: '):continue data=line[6:] if data==b'[DONE]':break v=json.loads(data) if v.get('usage'):usage=v['usage'] for c in v.get('choices',[]): delta=c.get('delta',{});content=delta.get('content') or '' if (content or delta.get('tool_calls')) and first is None:first=time.perf_counter() text+=content;finish=c.get('finish_reason') or finish for tc in delta.get('tool_calls') or []: idx=tc.get('index',0);dst=calls.setdefault(idx,{'id':'','type':'function','function':{'name':'','arguments':''}}) if tc.get('id'):dst['id']=tc['id'] for k in ['name','arguments']:dst['function'][k]+=tc.get('function',{}).get(k) or '' elapsed=time.perf_counter()-start return {'output':text,'tool_calls':list(calls.values()),'finish_reason':finish,'usage':usage,'elapsed_s':elapsed,'ttft_s':None if first is None else first-start,'tokens_per_s':usage.get('completion_tokens',0)/elapsed} def run_case(base,c): payload={'model':'occamy','messages':json.loads(json.dumps(c['messages'])),'temperature':0,'seed':42,'max_tokens':c['max_tokens'],'chat_template_kwargs':{'enable_thinking':False}} for msg in payload['messages']: if isinstance(msg['content'],str):msg['content']=unicodedata.normalize('NFC',msg['content']) if c.get('image'): p=payload['messages'][-1]['content'];data=base64.b64encode((R/c['image']).read_bytes()).decode();payload['messages'][-1]['content']=[{'type':'text','text':p},{'type':'image_url','image_url':{'url':'data:image/png;base64,'+data}}] if c['category']=='tool':payload.update(tools=TOOLS,tool_choice='auto') out={'id':c['id'],'category':c['category'],'expected':c['expect']} try: out.update(request(base,payload));s=out['output'].strip();e=c['expect'];cat=c['category'];passed=False if cat=='gsm8k': nums=re.findall(r'[-+]?\d[\d,]*(?:\.\d+)?',s.split('FINAL:')[-1]);out['parsed_answer']=nums[-1].replace(',','') if nums else None passed=bool(nums) and Decimal(out['parsed_answer'])==Decimal(e.replace(',','')) elif cat=='arc': out['strict_label_format']=s==e;match=re.match(r'^(?:Answer\s*:\s*)?([A-D1-4])(?:[.:)\s]|$)',s,re.I);passed=bool(match) and match.group(1).upper()==e elif cat=='json': try:passed=json.loads(s)==e except ValueError:pass try:out['json_content_after_fence_removal']=json.loads(re.sub(r'^```(?:json)?\s*|\s*```$','',s))==e except ValueError:out['json_content_after_fence_removal']=False elif cat=='code': res=subprocess.run([sys.executable,'-I',str(R/'code_check.py')],input=json.dumps({'code':s,'tests':e}),text=True,capture_output=True,timeout=6);out['code_check']={'returncode':res.returncode,'stdout':res.stdout,'stderr':res.stderr[-700:]};passed=res.returncode==0 and json.loads(res.stdout)['pass'] elif cat=='tool': calls=out['tool_calls'];valid=len(calls)==1 and calls[0]['function']['name']=='add' and json.loads(calls[0]['function']['arguments'])=={'a':e['a'],'b':e['b']};out['call_pass']=valid if valid: actual=json.loads(calls[0]['function']['arguments']);value=actual['a']+actual['b'];payload['messages'] += [{'role':'assistant','content':out['output'] or None,'tool_calls':calls},{'role':'tool','tool_call_id':calls[0]['id'],'content':str(value)}];payload['tool_choice']='none';follow=request(base,payload);out['followup']=follow answer_line=next((line for line in follow['output'].splitlines() if line.strip()),'');nums=re.findall(r'(?