"""Score prediction TSVs against eval labels with tools/eval_metrics.py (micro CER). Usage: score.py system= [system= ...] -- name:root:labels ... Every system must have ``/.tsv`` for every set; a missing or short file is an error, never a silent skip. """ from __future__ import annotations import json import sys from pathlib import Path from eval_metrics import evaluate def main() -> int: out = Path(sys.argv[1]) split = sys.argv.index("--") systems = dict(arg.split("=", 1) for arg in sys.argv[2:split]) sets = [arg.split(":") for arg in sys.argv[split + 1:]] report: dict[str, dict[str, dict]] = {} for name, _root, labels in sets: gold = {rel: text for rel, text in (line.split("\t", 1) for line in Path(labels).read_text(encoding="utf-8").splitlines() if "\t" in line)} for system, pred_dir in systems.items(): pred_file = Path(pred_dir) / f"{name}.tsv" preds = {} for line in pred_file.read_text(encoding="utf-8").splitlines(): parts = line.split("\t") preds[parts[0]] = parts[1] if len(parts) > 1 else "" missing = [rel for rel in gold if rel not in preds] if missing: raise RuntimeError(f"{system}/{name}: {len(missing)} crops have no prediction (e.g. {missing[0]})") report.setdefault(name, {})[system] = evaluate([(preds[rel], text) for rel, text in gold.items()]) out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8") names = list(systems) print(f"{'set':<22}" + "".join(f"{s:>30}" for s in names)) for name, per_system in report.items(): cells = [f"cp {m['cer_codepoint']:.4f} gr {m['cer_grapheme']:.4f} ex {m['exact_line_acc']:.2f}" for m in (per_system[s] for s in names)] print(f"{name:<22}" + "".join(f"{c:>30}" for c in cells) + f" n={per_system[names[0]]['n_lines']}") return 0 if __name__ == "__main__": sys.exit(main())