File size: 3,441 Bytes
5952424 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """Merge sharded restore_dsh.py outputs and print the three-way comparison.
Shards are stride slices, so each covers all gap lengths; merging re-weights each
length by its own sample count rather than averaging shard means (which would be
wrong whenever shards see different numbers of samples at a length).
The headline "Overall Average CER" in the DSH 41(1) comparison paper (2026) is the mean of the ten
per-length means, so that is what we report as CER_macro for comparability; the
sample-weighted CER_micro is printed alongside rather than chosen selectively.
python3 analysis/merge_dsh.py
"""
import glob, json, os
BASE = os.path.join(os.path.dirname(__file__), "dsh")
# Published reference numbers, the DSH 41(1) comparison paper (2026) DSH 41(1) 69-76, recent/unedited sets.
# Inscriptions verified reproducible from their released per-record predictions
# (top-1 51.28%, top-20 68.62% reproduced exactly; CER 0.2859 published).
# Ithaca and papyri rows were scored by us from the released per-record predictions
# using the verified metric port (our calculate_cer reproduces their stored CER with
# zero difference, and their published inscription top-1/top-20 to the decimal).
REF = {
"inscr": {
"Llama 3.1 8B (DSH 2026)": dict(CER=0.2859, top1=0.5128, top20=0.6862),
"Ithaca (Assael et al. 2022)": dict(CER=0.3118, top1=0.4841, top20=0.6067),
},
"pap": {
"Llama 3.1 8B (DSH 2026)": dict(CER=0.3010, top1=0.5037, top20=0.7051),
# Ithaca is inscriptions-only: no papyri row exists, by design not omission.
},
}
def merge(pattern):
per_len, tot, t1, t20 = {}, 0, 0.0, 0.0
files = sorted(glob.glob(pattern))
for f in files:
d = json.load(open(f))
for L, v in d.get("by_length", {}).items():
a = per_len.setdefault(int(L), dict(n=0, cer=0.0, t1=0.0, t20=0.0))
a["n"] += v["n"]
a["cer"] += v["CER"] * v["n"] # back out the sum from the mean
a["t1"] += v["top1"] * v["n"]
a["t20"] += v["top20"] * v["n"]
tot += d["n"]; t1 += d["top1"] * d["n"]; t20 += d["top20"] * d["n"]
if not per_len:
return None
by = {L: dict(n=a["n"], CER=a["cer"] / a["n"], top1=a["t1"] / a["n"],
top20=a["t20"] / a["n"]) for L, a in sorted(per_len.items())}
return dict(shards=len(files), n=tot,
CER_macro=sum(v["CER"] for v in by.values()) / len(by),
CER_micro=sum(v["CER"] * v["n"] for v in by.values()) / tot,
top1=t1 / tot, top20=t20 / tot, by_length=by)
for tag, name in (("inscr", "RECENT INSCRIPTIONS"), ("pap", "RECENT PAPYRI")):
m = merge(f"{BASE}/v4_recent_{tag}_s*.json")
print("=" * 72); print(name); print("=" * 72)
if not m:
print(" (no shard output yet)\n"); continue
print(f" shards={m['shards']} n={m['n']}")
print(f" {'system':<32}{'CER':>8}{'top-1':>9}{'top-20':>9}")
print(f" {'Stoicheia (ours, 405M)':<32}{m['CER_macro']:>8.4f}{m['top1']:>9.4f}{m['top20']:>9.4f}")
for k, v in REF[tag].items():
print(f" {k:<32}{v['CER']:>8.4f}{v['top1']:>9.4f}{v['top20']:>9.4f}")
print(f" (ours sample-weighted CER {m['CER_micro']:.4f}; macro is the comparable one)")
print(" per length:")
for L, v in m["by_length"].items():
print(f" L={L:<3} n={v['n']:<5} CER={v['CER']:.4f} top1={v['top1']:.4f} top20={v['top20']:.4f}")
print()
|