"""Evaluate standard IME tracks; never tune on a consumed blind release.""" import argparse import json import sys import time from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "src")) from vimeml.benchmarks.evaluate_ajimee import metrics, rerank, summarize from vimeml.benchmarks.standard_ime import read, load_standard_export, allow_evaluation, source_metrics from vimeml.training.data import write_json def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--benchmark", required=True, type=Path, help="One imported development/blind/regression directory") parser.add_argument("--checkpoint", required=True, type=Path) parser.add_argument("--tokenizer", type=Path, default=ROOT / "artifacts/tokenizers/ja-unigram-16k-v2") parser.add_argument("--output", required=True, type=Path) parser.add_argument("--device", choices=("cpu", "cuda"), default="cuda") parser.add_argument("--allow-draft-development", action="store_true") parser.add_argument("--evaluation-plan", type=Path) parser.add_argument("--cached-scores", type=Path, help="Reuse matching FP32 legacy scores; regression track only") parser.add_argument("--architecture", choices=("v2", "v4"), default="v2", help="v4 loads tiny_gpt_v4 with TF32 disabled; pass the matching 8K --tokenizer") args = parser.parse_args() manifest = read(args.benchmark / "manifest.json") plan = read(args.evaluation_plan) if args.evaluation_plan else None allow_evaluation(manifest, allow_draft=args.allow_draft_development, plan=plan, checkpoint=args.checkpoint) if args.output.exists(): raise ValueError("Preserve prior results; choose a new output.") receipt_path = args.benchmark.parent / "blind-consumption.json" if manifest["split"] == "blind": receipt = read(receipt_path) if receipt_path.exists() else {"plan_id": plan["plan_id"], "evaluations": []} if receipt["plan_id"] != plan["plan_id"] and plan.get("role") != "reference_test_after_consumption": raise ValueError("Blind release has been opened under a different plan; use a new held-out release.") model = str(args.checkpoint.resolve()) if any(r["checkpoint"] == model and r["status"] == "complete" for r in receipt["evaluations"]): raise ValueError("This planned model already has completed blind results; reuse them.") receipt["evaluations"].append({"checkpoint": model, "output": str(args.output.resolve()), "status": "started", "plan_id": plan["plan_id"], "result_role": plan.get("role", "blind"), "utc": datetime.now(timezone.utc).isoformat()}) write_json(receipt_path, receipt) started = time.perf_counter() manifest, provenance, rows = load_standard_export(args.benchmark) if args.cached_scores: if manifest["split"] != "regression": raise ValueError("Cache reuse is restricted to historical regression cases.") cached_report = read(args.cached_scores / "metrics.json") metadata = cached_report["model"] if metadata["precision"] != "fp32" or Path(metadata["checkpoint"]).resolve() != args.checkpoint.resolve(): raise ValueError("Cache model/precision does not match.") cached_rows = {r["id"]: r for r in (json.loads(line) for line in (args.cached_scores / "scores.jsonl").read_text(encoding="utf-8").splitlines())} for row in rows: cached = cached_rows[row["id"]] if (any(row[k] != cached[k] for k in ("query", "left_context", "answers")) or row["candidates"] != cached["candidates"] or row["orders"]["azookey"] != cached["orders"]["azookey"]): raise ValueError("Frozen legacy case/candidate pool differs from the cache.") row.update({k: cached[k] for k in ("orders", "fallbacks", "lm_scores")}) elif args.architecture == "v4": import torch from vimeml.training.infer_v4 import JapaneseLMV4 torch.set_num_threads(4) torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cudnn.allow_tf32 = False torch.set_float32_matmul_precision("highest") lm = JapaneseLMV4(args.checkpoint, args.tokenizer, args.device) metadata = lm.metadata if plan: expected = next(m for m in plan["models"] if m["checkpoint"] == str(args.checkpoint.resolve())) if metadata["checkpoint_step"] != expected["step"] or metadata["checkpoint_sha256"] != expected["sha256"]: raise ValueError("Checkpoint identity differs from the precommitted snapshot.") rerank(lm, rows) else: import torch from vimeml.training.infer import JapaneseLM torch.set_num_threads(4) lm = JapaneseLM(args.checkpoint, args.tokenizer, args.device, hash_checkpoint=False) metadata = lm.metadata if plan: expected = next(m for m in plan["models"] if m["checkpoint"] == str(args.checkpoint.resolve())) if (metadata["checkpoint_step"] != expected["step"] or lm.checkpoint_identity != {"config": expected["training_config"], "signatures": expected["signatures"]}): raise ValueError("Checkpoint identity differs from the precommitted snapshot.") rerank(lm, rows) args.output.mkdir(parents=True) (args.output / "scores.jsonl").write_text("".join(json.dumps(r, ensure_ascii=False)+"\n" for r in rows), encoding="utf-8") report = {"format": "vimeml_standard_ime_metrics_v1", "status": "complete", "model": metadata, "benchmark_manifest": manifest, "export_provenance": provenance, "metrics": summarize(rows), "source_metrics": source_metrics(rows, metrics), "elapsed_seconds": time.perf_counter()-started, "labels_formal_gold": manifest["labels_formal_gold"], "split": manifest["split"], "label_quality": manifest.get("label_quality", "unreviewed_draft"), "evaluation_plan": plan, "policy": manifest["score_policy"], "result_role": plan.get("role", "blind") if plan else manifest["split"], "cached_scores_reused": str(args.cached_scores) if args.cached_scores else None} write_json(args.output / "metrics.json", report) if manifest["split"] == "blind": receipt["evaluations"][-1]["status"] = "complete" write_json(receipt_path, receipt) print(json.dumps(report["source_metrics"], ensure_ascii=False, indent=2)) if __name__ == "__main__": main()