tinyquery/rescore.py

"""Apply the current evaluator to saved raw outputs without generating new answers."""
import argparse
from collections import defaultdict
import json
from pathlib import Path
from tinyquery.evaluate import score,aggregate


def rescore(data,predictions):
    rows={r['id']:r for r in map(json.loads,Path(data).read_text().splitlines())};path=Path(predictions)
    entries=[json.loads(l) for l in path.read_text().splitlines()];groups=defaultdict(list);changed=0
    for entry in entries:
        row=rows[entry['id']];metrics=score(row,entry['output']);changed+=metrics['success']!=entry['metrics']['success'];entry['metrics']=metrics
        for field in ['language','backend','operation']:groups[field+':'+row[field]].append(metrics)
    tmp=path.with_suffix('.tmp');tmp.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in entries));tmp.replace(path)
    summary_path=path.with_suffix('.summary.json');summary=json.loads(summary_path.read_text())
    summary['metrics']=aggregate([r['metrics'] for r in entries]);summary['groups']={k:aggregate(v) for k,v in groups.items()}
    summary['sql_metric']='Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.'
    summary['rescoring']='Same saved raw outputs, with explicit supplied-schema compilation; no generation or output repair.'
    summary_path.write_text(json.dumps(summary,indent=2));return {'predictions':str(path),'changed_success_labels':changed,'metrics':summary['metrics']}


def main():
    p=argparse.ArgumentParser();p.add_argument('--data',required=True);p.add_argument('--predictions',required=True)
    args=p.parse_args();print(json.dumps(rescore(args.data,args.predictions)))


if __name__=='__main__':main()