File size: 16,831 Bytes
c83c617 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 | """Local FP32 V4 comparison on explicitly frozen development/regression inputs.
No blind discovery, training, label editing, export, or scoring-rule tuning.
Candidate snapshots are recovered from historical completed evaluation records
when the original Standard export manifest is unavailable, with that limitation
recorded explicitly. Original files remain unchanged.
"""
import argparse
import copy
import gc
import json
import math
import re
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "src"))
from vimeml.benchmarks.ajimee import convert_items
from vimeml.benchmarks.comparison import paired, rows as read_rows
from vimeml.benchmarks.evaluate_ajimee import load_export, metrics, rerank, summarize
from vimeml.benchmarks.standard_ime import source_metrics
from vimeml.training.data import SentenceWindowDataset, file_sha, make_loader, write_json
V4 = ROOT / "artifacts/models/tiny-ja-v4-b-e8k-d384-l7"
BASELINE = ROOT / "artifacts/models/tiny-ja-v2.1-e16k-d320-l6-extend5/best.pt"
TRACKS = {
"standard-development": ("outputs/training-v3-b/reused-development/epoch-2",
"artifacts/benchmarks/ime-standard-ja-v1-ai-expert-r1/development", "development", 728),
"legacy-regression": ("outputs/training-v3-b/legacy500-selected",
"artifacts/benchmarks/ime-standard-ja-v1-ai-expert-r1/regression", "regression", 500),
"ajimee": ("outputs/ime-eval/tiny-ja-v2.1-extend5-best-ajimee",
"artifacts/benchmarks/ajimee-jwtd-v2-v1", "regression", 200),
"historical-development": ("outputs/ime-eval/tiny-ja-v2.1-extend5-best-development",
"artifacts/benchmarks/ime-dev-reviewed-v2", "development", 137),
}
def read(path):
return json.loads(Path(path).read_text(encoding="utf-8-sig"))
def frozen_row(row):
keep = ("position", "id", "source_index", "query", "left_context", "answers", "group",
"candidates", "source", "provenance")
result = {k: copy.deepcopy(row[k]) for k in keep if k in row}
result.update(orders={"azookey": list(row["orders"]["azookey"])}, fallbacks={})
if result["orders"]["azookey"] != [c["text"] for c in result["candidates"]]:
raise ValueError("Historical candidate order differs from candidate payload")
if len(set(result["orders"]["azookey"])) != len(result["candidates"]):
raise ValueError("Duplicate frozen candidate")
if any(not math.isfinite(c["score"]) for c in result["candidates"]):
raise ValueError("Nonfinite frozen AzooKey score")
return result
def prepare(output, model_directory=None, inventory_path=None):
if (output / "plan.json").exists():
plan = read(output / "plan.json")
if set(plan["tracks"]) != set(TRACKS) or plan.get("blind_used") is not False:
raise ValueError("Expected the explicit non-blind V4 evaluation plan")
if model_directory is not None and any(
(ROOT / model["path"]).parent != Path(model_directory).resolve()
for name, model in plan["models"].items() if name != "v21"
):
raise ValueError("Existing plan belongs to another model experiment")
for name, track in plan["tracks"].items():
cache, source, role, count = TRACKS[name]
allowed = {ROOT / cache / "scores.jsonl", ROOT / cache / "metrics.json",
ROOT / source / "evaluation_items.json", output / "inputs" / (name + ".json")}
if track["role"] != role or track["cases"] != count or {ROOT / p for p in track["input_hashes"]} != allowed:
raise ValueError("Plan differs from the allowlisted development/regression inputs")
for path, digest in track["input_hashes"].items():
if file_sha(ROOT / path) != digest:
raise ValueError(f"Frozen input changed: {path}")
return plan
output.mkdir(parents=True, exist_ok=True)
records = {}
for name, (cache, source, role, count) in TRACKS.items():
cache, source = ROOT / cache, ROOT / source
report = read(cache / "metrics.json")
if report["status"] != "complete":
raise ValueError("Incomplete historical source")
original = read_rows(cache / "scores.jsonl")
_, mapping, _ = convert_items(read(source / "evaluation_items.json"))
if len(original) != count or len(mapping) != count:
raise ValueError("Frozen sample count changed")
for row, expected in zip(original, mapping, strict=True):
if any(row[k] != expected[k] for k in expected):
raise ValueError("Historical candidate snapshot does not match retained labels/context")
if name in ("standard-development", "legacy-regression"):
if report["split"] != role or report["result_role"] != role:
raise ValueError("Only explicit development/regression records allowed")
for row, source_row in zip(original, read(source / "evaluation_items.json"), strict=True):
if row["source"] != source_row["provenance"]["source"] or row["provenance"] != source_row["provenance"]:
raise ValueError("Frozen source provenance differs")
provenance = "Exact historical candidate snapshot; original Standard export manifest unavailable locally"
elif name == "ajimee":
_, _, exported = load_export(source)
if [frozen_row(r) for r in exported] != [frozen_row(r) for r in original]:
raise ValueError("Cached pool differs from original retained AzooKey export")
provenance = "Original retained export verified by existing AJIMEE loader"
else:
if file_sha(source / "evaluation_items.json") != report["benchmark_manifest"]["source_sha256"]:
raise ValueError("Historical development label hash differs")
provenance = "Exact historical candidate snapshot verified against original report's retained reference SHA256; original candidate export unavailable locally"
snapshot = output / "inputs" / (name + ".json")
snapshot.parent.mkdir(exist_ok=True)
write_json(snapshot, [frozen_row(r) for r in original])
paths = [cache / "scores.jsonl", cache / "metrics.json", source / "evaluation_items.json", snapshot]
records[name] = {"cases": count, "role": role, "provenance": provenance,
"label_quality": report.get("label_quality", report["benchmark_manifest"].get("label_status", "historical_public_references")),
"labels_formal_gold": report.get("labels_formal_gold", False),
"input_hashes": {p.relative_to(ROOT).as_posix(): file_sha(p) for p in paths}}
models = {"v21": {"path": BASELINE.relative_to(ROOT).as_posix(), "sha256": file_sha(BASELINE)}}
model_directory = V4 if model_directory is None else Path(model_directory).resolve()
inventory_path = ROOT / "outputs/training-v4-20261015/remote-inventory.json" if inventory_path is None else Path(inventory_path).resolve()
inventory = read(inventory_path)
prefix = model_directory.relative_to(ROOT).as_posix() + "/"
epoch_names = sorted({Path(r["relative"]).stem for r in inventory
if r["relative"].startswith(prefix) and "/" not in r["relative"][len(prefix):]
and re.fullmatch(r"epoch-\d+\.pt", Path(r["relative"]).name)}, key=lambda n: int(n.split("-")[1]))
if not epoch_names:
raise ValueError("Expected recorded complete-epoch checkpoint identities")
for name in ["best", "last", "best-epoch", *epoch_names]:
relative = (model_directory / (name + ".pt")).relative_to(ROOT).as_posix()
record = next(r for r in inventory if r["relative"] == relative)
models[name] = {"path": relative, "sha256": record["sha256"]}
tokens = [read(ROOT / f"artifacts/token-data/{name}/manifest.json") for name in ("corpus-v2-16k", "corpus-v4-8k")]
if [(t["splits"]["validation"]["sentences"], t["splits"]["validation"]["characters"]) for t in tokens] != [(260337, 11110489)] * 2:
raise ValueError("Validation character/sentence populations differ")
# V2 rows.bin addresses JSONL; V4 rows.bin addresses the same frozen TXT.
# Their byte offsets differ by design. Compare original TXT identities instead.
validation_text_hashes = [{v for p, v in t["input_sha256"].items() if p.replace("\\", "/").endswith("/validation.txt")} for t in tokens]
if len(validation_text_hashes[0]) != 1 or validation_text_hashes[0] != validation_text_hashes[1]:
raise ValueError("Original validation TXT identities differ")
for suffix in ("sources.bin",):
if len({file_sha(ROOT / f"artifacts/token-data/{n}/validation.{suffix}") for n in ("corpus-v2-16k", "corpus-v4-8k")}) != 1:
raise ValueError("Validation row/source alignment differs")
plan = {"format": "vimeml_v4_local_evaluation_plan_v1", "created_utc": datetime.now(timezone.utc).isoformat(),
"tracks": records, "models": models, "blind_used": False, "test_split_used": False,
"epoch_checkpoints": epoch_names, "inventory_sha256": file_sha(inventory_path),
"primary_model": "best", "primary_comparator": "v21", "precision": "fp32; TF32 disabled",
"scoring": "Existing rerank(): joint-tokenized suffix full-vocabulary logP sum; no EOS/truncation; stable ties; whole-case fallback; full denominator",
"secondary": "context-free sum and length mean are diagnostics; no score coefficient tuning",
"development_selection": f"Highest Standard development contextual-sum Top-1 among best and {len(epoch_names)} complete epochs; ties prefer best, then earliest epoch. Exploratory selection, not held-out confirmation.",
"validation": "Same original 260337 sentences / 11110489 Unicode characters; all windows and prediction targets including EOS; compare BPC, not token PPL",
"generation": "Existing synthetic 6-token greedy/sample/beam settings; fixed 12 prompts with 16 generated tokens; character length reported because token budgets differ across vocabularies",
"statistics": "Exact paired McNemar descriptive p-values; multiple checkpoint/source comparisons exploratory, no unadjusted significance declarations"}
write_json(output / "plan.json", plan)
return plan
def load_model(name, plan, device):
from vimeml.training.infer import JapaneseLM
from vimeml.training.infer_v4 import JapaneseLMV4
path = ROOT / plan["models"][name]["path"]
if path.exists():
if file_sha(path) != plan["models"][name]["sha256"]:
raise ValueError("Original checkpoint hash changed")
else:
path = path.parent / "inference" / path.name
import torch
packed = torch.load(path, map_location="cpu", weights_only=True)
if packed.get("source_checkpoint_sha256") != plan["models"][name]["sha256"] or not packed.get("inference_only"):
raise ValueError("Inference checkpoint source binding differs")
del packed
tokenizer = ROOT / ("artifacts/tokenizers/ja-unigram-16k-v2" if name == "v21" else "artifacts/tokenizers/v4-a/ja-unigram-8192")
lm = (JapaneseLM if name == "v21" else JapaneseLMV4)(path, tokenizer, device)
lm.metadata.update(checkpoint=str(path), original_checkpoint_sha256=plan["models"][name]["sha256"],
device=str(device), tf32=False, parameters=lm.model.parameter_count())
return lm
def validation(lm, name, output):
from vimeml.training.train import evaluate
token = ROOT / "artifacts/token-data" / ("corpus-v2-16k" if name == "v21" else "corpus-v4-8k")
index = ROOT / "artifacts/training-data" / ("corpus-v2-c128" if name == "v21" else "corpus-v4-c128")
meta = read(token / "manifest.json")
for suffix in ("tokens.bin", "offsets.bin", "sources.bin"):
filename = f"validation.{suffix}"
if file_sha(token / filename) != meta["output_sha256"][filename]:
raise ValueError("Validation token file changed")
dataset = SentenceWindowDataset(token, index, "validation")
loader = make_loader(dataset, batch_size=128, num_workers=2, shuffle=False, pin_memory=lm.device.type == "cuda")
started = time.perf_counter()
result = evaluate(lm.model, loader, lm.device, "fp32", meta["splits"]["validation"]["characters"])
if result["prediction_pairs"] != meta["splits"]["validation"]["prediction_pairs"]:
raise ValueError("Full validation target coverage differs")
result.update(model=lm.metadata, sentences=meta["splits"]["validation"]["sentences"],
windows=len(dataset), token_manifest_sha256=file_sha(token / "manifest.json"),
elapsed_seconds=time.perf_counter() - started)
write_json(output / "validation.json", result)
dataset.close()
del loader
lm.model.eval()
print(name, "FULL_VALIDATION_BPC", result["bpc"], flush=True)
def generation(lm, output):
from vimeml.training import evaluate_ime as diagnostic
previous = diagnostic.JapaneseLM
try:
diagnostic.JapaneseLM = lambda *args, **kwargs: lm
diagnostic.main(["--device", str(lm.device), "--output", str(output / "synthetic")])
finally:
diagnostic.JapaneseLM = previous
samples = []
for prompt in read(ROOT / "configs/inference-prompts.json"):
results = [lm.generate(prompt["prompt"], max_new_tokens=16, temperature=temp, seed=42) for temp in (0, .8)]
for result in results:
result["new_characters"] = len(result["text"]) - len(prompt["prompt"])
samples.append({**prompt, "results": results})
write_json(output / "generation.json", {"model": lm.metadata, "samples": samples,
"qualitative_only": True, "prompt_sha256": file_sha(ROOT / "configs/inference-prompts.json")})
def run(name, output, plan, device, full=False, broad=False):
dest = output / name
dest.mkdir(exist_ok=True)
lm = load_model(name, plan, device)
for track in TRACKS if broad else ("standard-development",):
target = dest / track
if (target / "metrics.json").exists():
if read(target / "metrics.json")["model"]["original_checkpoint_sha256"] != lm.metadata["original_checkpoint_sha256"]:
raise ValueError("Existing evaluation model differs")
continue
inputs = read(output / "inputs" / (track + ".json"))
started = time.perf_counter()
rerank(lm, inputs)
target.mkdir(exist_ok=True)
(target / "scores.jsonl").write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in inputs), encoding="utf-8")
result = {"status": "complete", "model": lm.metadata, "input": plan["tracks"][track],
"metrics": summarize(inputs), "elapsed_seconds": time.perf_counter() - started}
if "source" in inputs[0]:
result["source_metrics"] = source_metrics(inputs, metrics)
baseline = output / "v21" / track / "scores.jsonl"
if name != "v21" and baseline.exists():
comparison = paired(read_rows(baseline), inputs)
comparison["paired"].pop("significant_at_0_05")
write_json(target / "paired-v21.json", comparison)
write_json(target / "metrics.json", result)
print(name, track, result["metrics"]["all"]["lm_context_sum"]["top1_correct"], flush=True)
if full and not (dest / "validation.json").exists():
validation(lm, name, dest)
if broad and not (dest / "generation.json").exists():
generation(lm, dest)
del lm
gc.collect()
import torch
if torch.cuda.is_available():
torch.cuda.empty_cache()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, default=ROOT / "outputs/evaluation-v4-20261015")
parser.add_argument("--name", default="plan", help="plan or a checkpoint name in the frozen inventory")
parser.add_argument("--model-directory", type=Path)
parser.add_argument("--inventory", type=Path)
parser.add_argument("--full-validation", action="store_true")
parser.add_argument("--broad", action="store_true")
parser.add_argument("--device", choices=("cpu", "cuda"), default="cuda")
args = parser.parse_args()
args.output = args.output.resolve()
import torch
torch.set_num_threads(4)
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
plan = prepare(args.output, args.model_directory, args.inventory)
if args.name != "plan":
if args.name not in plan["models"]:
parser.error("Checkpoint name is absent from the frozen inventory")
run(args.name, args.output, plan, args.device, args.full_validation, args.broad)
if __name__ == "__main__":
main()
|