Release unmodified quantization with complete saved evaluation metrics and historical reference
711ed9f verified | """Frozen legacy rubric with engine field fixes, frozen questions and isolated code execution.""" | |
| import pathlib,json,subprocess,uuid,os | |
| import legacy_capability as h | |
| from protocol import normalize_content | |
| ROOT=pathlib.Path(__file__).parent | |
| load_legacy=h.load_items | |
| def load_items(task,n,seed): | |
| data=json.loads((ROOT/'questions.json').read_text()) | |
| assert seed==data['seed'] | |
| items=data['tasks'][task] | |
| assert len(items)==n,(task,len(items),n) | |
| return items | |
| h.load_items=load_items | |
| # Expose vLLM's current reasoning field and finish reason without changing scoring. | |
| old_call=h.call | |
| def call(session,args,item,spec): | |
| class Adapter: | |
| def post(self,*a,**kw): | |
| r=session.post(*a,**kw) | |
| if r.ok: | |
| d=r.json();c=d['choices'][0];m=c['message'] | |
| raw=m.get('content') or '' | |
| content,reasoning,mode=normalize_content(raw,c.get('finish_reason')) if args.thinking else (raw,'','thinking-disabled') | |
| m['content']=content | |
| m['reasoning_content']=reasoning | |
| d.setdefault('usage',{})['content_mode']=mode | |
| d.setdefault('usage',{})['finish_reason']=c.get('finish_reason') | |
| r.json=lambda:d | |
| return r | |
| return old_call(Adapter(),args,item,spec) | |
| h.call=call | |
| def score_code(text,gold,timeout=15): | |
| code=h._extract_code(text) | |
| if not code.strip():return False | |
| program='import math,re,sys,itertools,collections,heapq,functools,string\n'+code+'\n'+gold['test']+'\ncheck('+gold['entry_point']+')\n' | |
| name='veriloop-eval-'+uuid.uuid4().hex | |
| cmd=['docker','run','--rm','-i','--name',name,'--network','none','--read-only','--cap-drop','ALL','--security-opt','no-new-privileges','--pids-limit','64','--memory','256m','--cpus','1','--user','65534:65534','--entrypoint','python3','flashnext-base:locked-20261008','-I','-'] | |
| try:return subprocess.run(cmd,input=program,text=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=timeout).returncode==0 | |
| except subprocess.TimeoutExpired:return False | |
| finally:subprocess.run(['docker','rm','-f',name],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) | |
| h.score_code=score_code | |
| def main(): | |
| import argparse,threading,time | |
| from concurrent.futures import ThreadPoolExecutor | |
| p=argparse.ArgumentParser() | |
| for flag in ['base-url','model','label','outdir']:p.add_argument('--'+flag,required=True) | |
| p.add_argument('--tasks',default='all');p.add_argument('--seed',type=int,default=20260816) | |
| p.add_argument('--max-tokens',dest='max_tokens_override',type=int,default=32768) | |
| p.add_argument('--concurrency',type=int,default=16);p.add_argument('--timeout',type=int,default=3600) | |
| p.add_argument('--no-thinking',dest='thinking',action='store_false') | |
| args=p.parse_args();args.n_override=0;args.temperature=0.;args.top_p=1.;args.retries=3 | |
| tasks=h.ALL_TASKS if args.tasks=='all' else args.tasks.split(',') | |
| limit=threading.BoundedSemaphore(args.concurrency) | |
| underlying=h.call | |
| def bounded(*a,**kw): | |
| with limit:return underlying(*a,**kw) | |
| h.call=bounded | |
| out=pathlib.Path(args.outdir);out.mkdir(exist_ok=True) | |
| summary={'label':args.label,'model':args.model,'base_url':args.base_url,'seed':args.seed,'thinking':args.thinking,'started':time.strftime('%F %T'),'tasks':{},'scheduler':'task workers share one request semaphore'} | |
| def task(t): | |
| ckpt=out/(args.label+'-'+t+'.json') | |
| if ckpt.exists():r=json.loads(ckpt.read_text()) | |
| else: | |
| print('START',t,flush=True);r=h.run_task(args,t) | |
| if r['errors']:raise RuntimeError(t+' request errors') | |
| ckpt.write_text(json.dumps(r,ensure_ascii=False,indent=2)) | |
| print('RESULT',t,r['hits'],r['n'],'empty',r.get('empty'),'length',r.get('length_limited'),flush=True) | |
| return t,{k:v for k,v in r.items() if k!='details'} | |
| with ThreadPoolExecutor(max_workers=len(tasks)) as pool: | |
| for t,r in pool.map(task,tasks):summary['tasks'][t]=r | |
| summary['finished']=time.strftime('%F %T') | |
| (out/('SUMMARY-'+args.label+'.json')).write_text(json.dumps(summary,indent=2)) | |
| if __name__=='__main__':main() | |