File size: 3,707 Bytes
c83c617 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | """V2 sentence-complete validation subsets and frozen IME epoch diagnostics."""
import json
import math
from pathlib import Path
import numpy as np
import sentencepiece as spm
from vimeml.training.data import write_json
def validation_subset(dataset, sampled_indices, tokenizer_dir):
"""Include all windows of sampled sentences so character counts are exact."""
processor = spm.SentencePieceProcessor(model_file=str(Path(tokenizer_dir) / "tokenizer.model"))
sentences = sorted({dataset.locate(index)[0] for index in sampled_indices})
indices = []
characters = 0
dataset._open()
try:
for sentence in sentences:
tokens = dataset._store[sentence]
characters += len(processor.decode(tokens[1:-1]))
position = int(np.searchsorted(dataset.sentences, sentence, side="left"))
extra = 0 if position == 0 else int(dataset.end[position - 1] - dataset.sentences[position - 1] - 1)
first = sentence + extra
indices.extend(
range(first, first + math.ceil((len(tokens) - 1) / dataset.context_length))
)
finally:
dataset.close()
return indices, characters
def epoch_ime(model, tokenizer_dir, benchmarks, root, output, epoch_number):
from vimeml.benchmarks.evaluate_ajimee import load_export, metrics, rerank, summarize
from vimeml.benchmarks.standard_ime import FORMAT, allow_evaluation, load_standard_export, source_metrics
from vimeml.training.infer import JapaneseLM
lm = JapaneseLM.__new__(JapaneseLM)
lm.model = model
lm.device = next(model.parameters()).device
lm.processor = spm.SentencePieceProcessor(
model_file=str(Path(tokenizer_dir) / "tokenizer.model")
)
lm.special = {
name: getattr(lm.processor, f"{name}_id")() for name in ("pad", "unk", "bos", "eos")
}
reports = {}
model.eval()
try:
for name, directory in benchmarks.items():
path = Path(root) / directory
header = json.loads((path / "manifest.json").read_text(encoding="utf-8"))
standard = header.get("format") == FORMAT
if standard:
if header.get("split") != "development":
raise ValueError("Training diagnostics may only open standard IME development")
allow_evaluation(header)
manifest, provenance, rows = load_standard_export(path)
else:
manifest, provenance, rows = load_export(path)
rerank(lm, rows)
report = {
"epoch": epoch_number,
"benchmark": directory,
"benchmark_manifest": manifest,
"export_provenance": provenance,
"metrics": summarize(rows),
"policy": "Frozen joint tokenization; full-vocabulary suffix logP sum; stable ties; no EOS.",
}
if standard:
report.update(source_metrics=source_metrics(rows, metrics),
label_quality=manifest["label_quality"],
labels_formal_gold=manifest["labels_formal_gold"])
destination = Path(output) / "epoch-evaluation" / f"epoch-{epoch_number}" / name
destination.mkdir(parents=True, exist_ok=True)
write_json(destination / "metrics.json", report)
with (destination / "scores.jsonl").open("w", encoding="utf-8") as stream:
for row in rows:
stream.write(json.dumps(row, ensure_ascii=False) + "\n")
reports[name] = report["metrics"]["all"]["lm_context_sum"]
finally:
model.train()
return reports
|