File size: 2,029 Bytes
d82db76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
"""Score prediction TSVs against eval labels with tools/eval_metrics.py (micro CER).

Usage: score.py <out.json> system=<pred_dir> [system=<pred_dir> ...] -- name:root:labels ...
Every system must have ``<pred_dir>/<name>.tsv`` for every set; a missing or short file is
an error, never a silent skip.
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

from eval_metrics import evaluate


def main() -> int:
    out = Path(sys.argv[1])
    split = sys.argv.index("--")
    systems = dict(arg.split("=", 1) for arg in sys.argv[2:split])
    sets = [arg.split(":") for arg in sys.argv[split + 1:]]
    report: dict[str, dict[str, dict]] = {}
    for name, _root, labels in sets:
        gold = {rel: text for rel, text in (line.split("\t", 1) for line in Path(labels).read_text(encoding="utf-8").splitlines() if "\t" in line)}
        for system, pred_dir in systems.items():
            pred_file = Path(pred_dir) / f"{name}.tsv"
            preds = {}
            for line in pred_file.read_text(encoding="utf-8").splitlines():
                parts = line.split("\t")
                preds[parts[0]] = parts[1] if len(parts) > 1 else ""
            missing = [rel for rel in gold if rel not in preds]
            if missing:
                raise RuntimeError(f"{system}/{name}: {len(missing)} crops have no prediction (e.g. {missing[0]})")
            report.setdefault(name, {})[system] = evaluate([(preds[rel], text) for rel, text in gold.items()])
    out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
    names = list(systems)
    print(f"{'set':<22}" + "".join(f"{s:>30}" for s in names))
    for name, per_system in report.items():
        cells = [f"cp {m['cer_codepoint']:.4f} gr {m['cer_grapheme']:.4f} ex {m['exact_line_acc']:.2f}" for m in (per_system[s] for s in names)]
        print(f"{name:<22}" + "".join(f"{c:>30}" for c in cells) + f"   n={per_system[names[0]]['n_lines']}")
    return 0


if __name__ == "__main__":
    sys.exit(main())