| """Reproduce frozen V4.1 FP32 comparisons in a separate local output directory. |
| |
| Requires the hash-verified model download and the four pinned nonblind pools. |
| The original October 16 evaluation script remains with its recorded outputs. |
| """ |
| import argparse |
| import copy |
| import datetime |
| import gc |
| import json |
| import sys |
| import time |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[2] |
| sys.path.insert(0, str(ROOT / 'src')) |
| import torch |
| from vimeml.training.data import file_sha, write_json |
| from vimeml.training.infer_v4 import JapaneseLMV4 |
| from vimeml.benchmarks.evaluate_ajimee import rerank, metrics, summarize |
| from vimeml.benchmarks.standard_ime import source_metrics |
| from vimeml.benchmarks.comparison import paired, rows as read_rows |
| from vimeml.posttraining.infer import TRACKS, read_snapshot, load_conditional_checkpoint, fp32_evaluation, evaluate_snapshot |
|
|
| OUT = ROOT / 'outputs/evaluation-v41-20261016' |
| MODEL = ROOT / 'artifacts/models/tiny-ja-v41-e8k-d384-l7' |
| TOKENIZER = ROOT / 'artifacts/tokenizers/v4-a/ja-unigram-8192' |
| BASELINES = {'v21': ROOT / 'outputs/evaluation-v4-20261015/v21', |
| 'v4b': ROOT / 'outputs/evaluation-v4-b12cosine-20261015/best'} |
|
|
|
|
| def read(path): |
| return json.loads(path.read_text(encoding='utf-8')) |
|
|
|
|
| def save_rows(path, rows): |
| path.parent.mkdir(parents=True, exist_ok=True) |
| path.write_text(''.join(json.dumps(r, ensure_ascii=False) + '\n' for r in rows), encoding='utf-8') |
|
|
|
|
| def check_base(base, frozen): |
| assert len(base) == len(frozen) |
| for a, b in zip(base, frozen, strict=True): |
| for k in ['id', 'query', 'left_context', 'answers', 'candidates']: |
| assert a[k] == b[k], (k, b['id']) |
| assert a['orders']['azookey'] == b['orders']['azookey'] |
|
|
|
|
| def comparisons(destination, scored, track): |
| for name, folder in BASELINES.items(): |
| base = read_rows(folder / track / 'scores.jsonl') |
| check_base(base, scored) |
| result = paired(base, scored) |
| result['paired'].pop('significant_at_0_05', None) |
| write_json(destination / f'paired-{name}.json', result) |
|
|
|
|
| def replay_check(name, scored, conditional=False): |
| epoch = 13 if name == 'best-ordinary' else 16 |
| folder = 'standard-development' if conditional else 'standard-ordinary' |
| remote = {r['id']: r for r in read_rows(MODEL / f'epoch-evaluation/epoch-{epoch}/{folder}/scores.jsonl')} |
| differences, top1_differences = [], [] |
| for r in scored: |
| if r['orders']['lm_context_sum'] != remote[r['id']]['ranked']: |
| differences.append(r['id']) |
| if r['orders']['lm_context_sum'][0] != remote[r['id']]['ranked'][0]: |
| top1_differences.append(r['id']) |
| result = {'epoch': epoch, 'cases': len(scored), 'complete_order_differences': differences, |
| 'top1_choice_differences': top1_differences, |
| 'top1_correct': metrics(scored, 'lm_context_sum')['top1_correct'], |
| 'remote_top1_correct': sum(r['correct'] for r in remote.values())} |
| if result['top1_correct'] != result['remote_top1_correct']: |
| raise ValueError(('Local FP32 Top-1 differs from frozen training-time result', result)) |
| return result |
|
|
|
|
| def ordinary(name, inputs): |
| dest = OUT / name / 'ordinary' |
| lm = JapaneseLMV4(MODEL / f'{name}.pt', TOKENIZER, 'cuda') |
| reports = {} |
| for track, frozen in inputs.items(): |
| target = dest / track |
| if (target / 'metrics.json').exists(): |
| reports[track] = read(target / 'metrics.json') |
| continue |
| rows = copy.deepcopy(frozen) |
| began = time.perf_counter() |
| rerank(lm, rows) |
| result = {'model': lm.metadata, 'metrics': summarize(rows), |
| 'elapsed_seconds': time.perf_counter() - began, |
| 'source_metrics': source_metrics(rows, metrics) if 'source' in rows[0] else None} |
| if track == 'standard-development': |
| result['remote_replay'] = replay_check(name, rows) |
| save_rows(target / 'scores.jsonl', rows) |
| comparisons(target, rows, track) |
| write_json(target / 'metrics.json', result) |
| reports[track] = result |
| print(name, 'ordinary', track, result['metrics']['all']['lm_context_sum']['top1_correct'], flush=True) |
| generated = [] |
| with torch.inference_mode(): |
| for prompt in read(ROOT / 'configs/inference-prompts.json'): |
| generated.append({**prompt, 'results': [lm.generate(prompt['prompt'], max_new_tokens=16, temperature=t, seed=42) for t in [0, .8]]}) |
| write_json(dest / 'generation.json', {'qualitative_only': True, 'prompts_sha256': file_sha(ROOT / 'configs/inference-prompts.json'), 'samples': generated}) |
| del lm |
| gc.collect() |
| torch.cuda.empty_cache() |
| return reports |
|
|
|
|
| def conditional(inputs): |
| model, codec, metadata = load_conditional_checkpoint(MODEL / 'best-rerank.pt', TOKENIZER / 'tokenizer.model', 'cuda') |
| reports = {} |
| with fp32_evaluation(model): |
| for condition in ['context_reading', 'empty_reading', 'empty_context']: |
| reports[condition] = {} |
| for track, frozen in inputs.items(): |
| if condition != 'context_reading' and track != 'standard-development': |
| continue |
| target = OUT / 'best-rerank' / condition / track |
| if (target / 'comparison-metrics.json').exists(): |
| reports[condition][track] = read(target / 'comparison-metrics.json') |
| continue |
| began = time.perf_counter() |
| result = evaluate_snapshot(model, codec, frozen, target, condition) |
| records = read_rows(target / 'scores.jsonl') |
| common = copy.deepcopy(frozen) |
| for row, record in zip(common, records, strict=True): |
| assert row['id'] == record['id'] and row['answers'] == record['answers'] |
| assert row['orders']['azookey'] == record['candidate_order'] |
| row['orders']['lm_context_sum'] = record['ranked'] |
| if record['fallback']: |
| row['fallbacks']['lm_context_sum'] = record['fallback'] |
| report = {'model': metadata, 'condition': condition, 'raw_metrics': result, |
| 'metrics': summarize(common), 'elapsed_seconds': time.perf_counter() - began, |
| 'source_metrics': source_metrics(common, metrics) if 'source' in common[0] else None} |
| if condition == 'context_reading' and track == 'standard-development': |
| report['remote_replay'] = replay_check('last', common, conditional=True) |
| save_rows(target / 'comparison-scores.jsonl', common) |
| comparisons(target, common, track) |
| write_json(target / 'comparison-metrics.json', report) |
| reports[condition][track] = report |
| print('best-rerank', condition, track, result['correct'], flush=True) |
| del model |
| gc.collect() |
| torch.cuda.empty_cache() |
| return reports |
|
|
|
|
| def main(): |
| global OUT |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument('--output', type=Path, required=True, |
| help='Separate output directory; completed results are never overwritten') |
| args = parser.parse_args() |
| OUT = args.output.resolve() |
| if OUT.exists() and any(OUT.iterdir()): |
| parser.error('Use an empty output directory to preserve prior evaluations') |
| if not torch.cuda.is_available(): |
| parser.error('The recorded V4.1 comparison uses an FP32 CUDA GPU') |
| receipt = read(ROOT / 'outputs/training-v41-20261016/download-receipt.json') |
| assert receipt['status'] == 'verified' |
| inputs = {track: read_snapshot(track) for track in TRACKS} |
| hashes = {r['relative']: r['sha256'] for r in receipt['files']} |
| for name in ['best-ordinary', 'last', 'best-rerank']: |
| p = MODEL / f'{name}.pt' |
| assert file_sha(p) == hashes[p.relative_to(ROOT).as_posix()] |
| OUT.mkdir(parents=True, exist_ok=True) |
| plan = {'created_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), |
| 'primary': 'best-ordinary.pt selected on Standard development at epoch13 before these local comparisons', |
| 'secondary': 'last.pt epoch16; best-rerank.pt epoch16; conditional empty-READ/CTX on Standard only; no coefficients tuned', |
| 'tracks': {k: {'cases': len(v), 'sha256': TRACKS[k][1]} for k, v in inputs.items()}, |
| 'checkpoints': {n: file_sha(MODEL / f'{n}.pt') for n in ['best-ordinary', 'last', 'best-rerank']}, |
| 'precision': 'primitive FP32; TF32 disabled; no quantization', |
| 'statistics': 'Exact paired McNemar descriptive p-values; correlated development/regression tracks, repeated checkpoint selection; no independent-test or multiplicity-adjusted claims', |
| 'blind_read': False, 'test_read': False, 'script_sha256': file_sha(Path(__file__))} |
| if (OUT / 'plan.json').exists(): |
| previous = read(OUT / 'plan.json') |
| assert previous['checkpoints'] == plan['checkpoints'] and previous['script_sha256'] == plan['script_sha256'] |
| else: |
| write_json(OUT / 'plan.json', plan) |
| (OUT / 'evaluation-script.py').write_bytes(Path(__file__).read_bytes()) |
| torch.set_num_threads(4) |
| torch.backends.cuda.matmul.allow_tf32 = False |
| torch.backends.cudnn.allow_tf32 = False |
| torch.set_float32_matmul_precision('highest') |
| write_json(OUT / 'environment.json', {'torch': str(torch.__version__), 'cuda': torch.version.cuda, |
| 'gpu': torch.cuda.get_device_name(), 'precision': 'FP32', 'tf32': False}) |
| results = {} |
| with torch.inference_mode(): |
| for name in ['best-ordinary', 'last']: |
| results[name] = ordinary(name, inputs) |
| results['conditional'] = conditional(inputs) |
| for track in TRACKS: |
| read_snapshot(track) |
| write_json(OUT / 'summary.json', {'status': 'complete', 'results': results, 'blind_read': False, 'test_read': False}) |
| print('LOCAL_ANALYSIS_COMPLETE', flush=True) |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
