File size: 2,029 Bytes
d82db76 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | """Score prediction TSVs against eval labels with tools/eval_metrics.py (micro CER).
Usage: score.py <out.json> system=<pred_dir> [system=<pred_dir> ...] -- name:root:labels ...
Every system must have ``<pred_dir>/<name>.tsv`` for every set; a missing or short file is
an error, never a silent skip.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
from eval_metrics import evaluate
def main() -> int:
out = Path(sys.argv[1])
split = sys.argv.index("--")
systems = dict(arg.split("=", 1) for arg in sys.argv[2:split])
sets = [arg.split(":") for arg in sys.argv[split + 1:]]
report: dict[str, dict[str, dict]] = {}
for name, _root, labels in sets:
gold = {rel: text for rel, text in (line.split("\t", 1) for line in Path(labels).read_text(encoding="utf-8").splitlines() if "\t" in line)}
for system, pred_dir in systems.items():
pred_file = Path(pred_dir) / f"{name}.tsv"
preds = {}
for line in pred_file.read_text(encoding="utf-8").splitlines():
parts = line.split("\t")
preds[parts[0]] = parts[1] if len(parts) > 1 else ""
missing = [rel for rel in gold if rel not in preds]
if missing:
raise RuntimeError(f"{system}/{name}: {len(missing)} crops have no prediction (e.g. {missing[0]})")
report.setdefault(name, {})[system] = evaluate([(preds[rel], text) for rel, text in gold.items()])
out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
names = list(systems)
print(f"{'set':<22}" + "".join(f"{s:>30}" for s in names))
for name, per_system in report.items():
cells = [f"cp {m['cer_codepoint']:.4f} gr {m['cer_grapheme']:.4f} ex {m['exact_line_acc']:.2f}" for m in (per_system[s] for s in names)]
print(f"{name:<22}" + "".join(f"{c:>30}" for c in cells) + f" n={per_system[names[0]]['n_lines']}")
return 0
if __name__ == "__main__":
sys.exit(main())
|