| """Sequential isolated-process checkpoint comparison. Never promotes a model.""" |
| import json |
| from pathlib import Path |
| import subprocess |
| import sys |
|
|
|
|
| def run(module,args,log): |
| result = subprocess.run([sys.executable,'-m',module,*args],stdout=subprocess.PIPE, |
| stderr=subprocess.STDOUT,text=True,encoding='utf-8',errors='replace') |
| Path(log).write_text(result.stdout,encoding='utf-8') |
| if result.returncode: |
| raise RuntimeError(f'{module} failed; inspect {log}') |
|
|
|
|
| def main(): |
| Path('reports').mkdir(exist_ok=True) |
| candidates = [('v000-mean',False),('v001-no-lexical',False),('v002-gru',False), |
| ('v003-transformer',False),('v000-mean',True)] |
| ranked = [] |
| for name,quantized in candidates: |
| label = name + ('-int8' if quantized else '-fp32') |
| output = f'reports/bench-{label}.json' |
| run('baim.bench_policy',['--checkpoint',f'models/{name}','--output',output] |
| + (['--quantized'] if quantized else []),f'reports/bench-{label}.log') |
| report = json.loads(Path(output).read_text()) |
| evaluation = report['evaluation'] |
| |
| score = evaluation['test']['joint_step_accuracy'] * evaluation['novel_wording']['joint_step_accuracy'] / ( |
| max(report['end_to_end_policy_ms']['p95'],.001) * max(report['observed_process_rss_bytes']/1024**3,.01)) |
| ranked.append(dict(label=label,diagnostic_score=score,report=output, |
| test_joint=evaluation['test']['joint_step_accuracy'],novel_joint=evaluation['novel_wording']['joint_step_accuracy'], |
| p95_policy_ms=report['end_to_end_policy_ms']['p95'],rss_bytes=report['observed_process_rss_bytes'], |
| disk_bytes=report['disk_bytes'])) |
| print(json.dumps(ranked[-1]),flush=True) |
| ranked.sort(key=lambda row:row['diagnostic_score'],reverse=True) |
| summary = dict(candidates=ranked,production_promoted=False, |
| formula='test_joint * novel_wording_joint / (p95_policy_ms * observed_python_rss_GiB)', |
| reason='Diagnostic synthetic ranking only; target-hardware, full task, recovery and security evidence missing.') |
| Path('reports/model-comparison.json').write_text(json.dumps(summary,indent=2),encoding='utf-8') |
|
|
|
|
| if __name__=='__main__': |
| main() |
|
|
