occamy-1.0-FP8 / evaluation /evaluate.py
Eang's picture
Release Occamy 1.0 FP8
ac48ca6
Raw History Blame Contribute Delete
5.8 kB
import json,time,requests,base64,re,subprocess,sys,unicodedata,concurrent.futures,statistics
from pathlib import Path
from decimal import Decimal
R=Path(__file__).parent
TOOLS=[{'type':'function','function':{'name':'add','description':'Add two integers','parameters':{'type':'object','properties':{'a':{'type':'integer'},'b':{'type':'integer'}},'required':['a','b']}}}]
def request(base,payload):
payload=dict(payload,stream=True,stream_options={'include_usage':True});start=time.perf_counter();first=None;text='';usage={};finish=None;calls={}
with requests.post(base+'/v1/chat/completions',json=payload,stream=True,timeout=(15,360)) as r:
r.raise_for_status()
for line in r.iter_lines():
if not line.startswith(b'data: '):continue
data=line[6:]
if data==b'[DONE]':break
v=json.loads(data)
if v.get('usage'):usage=v['usage']
for c in v.get('choices',[]):
delta=c.get('delta',{});content=delta.get('content') or ''
if (content or delta.get('tool_calls')) and first is None:first=time.perf_counter()
text+=content;finish=c.get('finish_reason') or finish
for tc in delta.get('tool_calls') or []:
idx=tc.get('index',0);dst=calls.setdefault(idx,{'id':'','type':'function','function':{'name':'','arguments':''}})
if tc.get('id'):dst['id']=tc['id']
for k in ['name','arguments']:dst['function'][k]+=tc.get('function',{}).get(k) or ''
elapsed=time.perf_counter()-start
return {'output':text,'tool_calls':list(calls.values()),'finish_reason':finish,'usage':usage,'elapsed_s':elapsed,'ttft_s':None if first is None else first-start,'tokens_per_s':usage.get('completion_tokens',0)/elapsed}
def run_case(base,c):
payload={'model':'occamy','messages':json.loads(json.dumps(c['messages'])),'temperature':0,'seed':42,'max_tokens':c['max_tokens'],'chat_template_kwargs':{'enable_thinking':False}}
for msg in payload['messages']:
if isinstance(msg['content'],str):msg['content']=unicodedata.normalize('NFC',msg['content'])
if c.get('image'):
p=payload['messages'][-1]['content'];data=base64.b64encode((R/c['image']).read_bytes()).decode();payload['messages'][-1]['content']=[{'type':'text','text':p},{'type':'image_url','image_url':{'url':'data:image/png;base64,'+data}}]
if c['category']=='tool':payload.update(tools=TOOLS,tool_choice='auto')
out={'id':c['id'],'category':c['category'],'expected':c['expect']}
try:
out.update(request(base,payload));s=out['output'].strip();e=c['expect'];cat=c['category'];passed=False
if cat=='gsm8k':
nums=re.findall(r'[-+]?\d[\d,]*(?:\.\d+)?',s.split('FINAL:')[-1]);out['parsed_answer']=nums[-1].replace(',','') if nums else None
passed=bool(nums) and Decimal(out['parsed_answer'])==Decimal(e.replace(',',''))
elif cat=='arc':
out['strict_label_format']=s==e;match=re.match(r'^(?:Answer\s*:\s*)?([A-D1-4])(?:[.:)\s]|$)',s,re.I);passed=bool(match) and match.group(1).upper()==e
elif cat=='json':
try:passed=json.loads(s)==e
except ValueError:pass
try:out['json_content_after_fence_removal']=json.loads(re.sub(r'^```(?:json)?\s*|\s*```$','',s))==e
except ValueError:out['json_content_after_fence_removal']=False
elif cat=='code':
res=subprocess.run([sys.executable,'-I',str(R/'code_check.py')],input=json.dumps({'code':s,'tests':e}),text=True,capture_output=True,timeout=6);out['code_check']={'returncode':res.returncode,'stdout':res.stdout,'stderr':res.stderr[-700:]};passed=res.returncode==0 and json.loads(res.stdout)['pass']
elif cat=='tool':
calls=out['tool_calls'];valid=len(calls)==1 and calls[0]['function']['name']=='add' and json.loads(calls[0]['function']['arguments'])=={'a':e['a'],'b':e['b']};out['call_pass']=valid
if valid:
actual=json.loads(calls[0]['function']['arguments']);value=actual['a']+actual['b'];payload['messages'] += [{'role':'assistant','content':out['output'] or None,'tool_calls':calls},{'role':'tool','tool_call_id':calls[0]['id'],'content':str(value)}];payload['tool_choice']='none';follow=request(base,payload);out['followup']=follow
answer_line=next((line for line in follow['output'].splitlines() if line.strip()),'');nums=re.findall(r'(?<![\d])[-+]?\d+',answer_line);out['followup_answer_line']=answer_line;passed=bool(nums) and int(nums[-1])==value and follow['finish_reason']=='stop'
elif cat=='vision':passed=s.lower().strip('.! ')==str(e).lower()
else:passed=s==e
out['pass']=bool(passed);out['truncated']=out['finish_reason']=='length'
except Exception as exc:out.update(error=repr(exc),pass_=False);out['pass']=False
return out
def evaluate(base,name):
cases=json.loads((R/'cases.json').read_text());path=R/(name+'.jsonl');done={}
if path.exists():done={x['id']:x for x in map(json.loads,path.read_text().splitlines())}
request(base,{'model':'occamy','messages':[{'role':'user','content':'Say hello.'}],'max_tokens':16,'temperature':0,'chat_template_kwargs':{'enable_thinking':False}})
with path.open('a') as f:
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
futures=[pool.submit(run_case,base,c) for c in cases if c['id'] not in done]
for fut in concurrent.futures.as_completed(futures):
x=fut.result();f.write(json.dumps(x,ensure_ascii=False)+'\n');f.flush();done[x['id']]=x
if len(done)%8==0:print(name,len(done),'/',len(cases),'passed',sum(v['pass'] for v in done.values()),flush=True)
# Separate short sequential timing probes; report shared hardware caveat.
perf=[]
for _ in range(3):perf.append(request(base,{'model':'occamy','messages':[{'role':'user','content':'Explain in about 80 words how binary search works.'}],'max_tokens':128,'temperature':0,'chat_template_kwargs':{'enable_thinking':False}}))
(R/(name+'-performance.json')).write_text(json.dumps(perf,indent=2));print(name,'DONE',flush=True)
if __name__=='__main__':evaluate(sys.argv[1],sys.argv[2])