ocr-tech-k2 / tests /test_references_validation.py
prosperious's picture
Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified
Raw History Blame Contribute Delete
27.3 kB
from __future__ import annotations
import json
from pathlib import Path
from ocrtech.benchmark import run_benchmark
from ocrtech.references import load_reference, score_document
from ocrtech.manifest import ManifestEntry, sha256_file, sha256_text, write_manifest
from ocrtech.references import materialize_references
from ocrtech.schemas import BBox, Block, Document, Page, Table, TableCell
from ocrtech.validation import check_leakage, validate_claim
def test_benchmark_scores_reference_json_structure(tmp_path: Path) -> None:
source = tmp_path / "page.txt"
source.write_text(
"नेपाल सरकार\n"
"नाम उमेर जिल्ला\n"
"राम ३२ काठमाडौं\n"
"Sita 29 Lalitpur\n",
encoding="utf-8",
)
source.with_suffix(".ref.json").write_text(
json.dumps(
{
"text": "नेपाल सरकार",
"reading_order": ["p0-l0", "p0-l1", "p0-l2", "p0-l3"],
"tables": [
[
["नाम", "उमेर", "जिल्ला"],
["राम", "३२", "काठमाडौं"],
["Sita", "29", "Lalitpur"],
]
],
},
ensure_ascii=False,
),
encoding="utf-8",
)
results = run_benchmark([source], tmp_path / "bench", baselines=["ours"])
assert results[0].metrics["reading_order_pair_accuracy"] == 1.0
assert results[0].metrics["table_cell_f1"] == 1.0
assert results[0].metrics["table_structure_similarity"] == 1.0
assert "text_cer" in results[0].metrics
def test_benchmark_scores_document_reconstruction_markdown(tmp_path: Path) -> None:
source = tmp_path / "page.txt"
source.write_text(
"नेपाल सरकार\n"
"नाम उमेर जिल्ला\n"
"राम ३२ काठमाडौं\n"
"Sita 29 Lalitpur\n",
encoding="utf-8",
)
source.with_suffix(".ref.json").write_text(
json.dumps(
{
"markdown": "# नेपाल सरकार\n\n| नाम | उमेर | जिल्ला |\n| --- | --- | --- |\n| राम | ३२ | काठमाडौं |\n| Sita | 29 | Lalitpur |\n",
"tables": [
[
["नाम", "उमेर", "जिल्ला"],
["राम", "३२", "काठमाडौं"],
["Sita", "29", "Lalitpur"],
]
],
},
ensure_ascii=False,
),
encoding="utf-8",
)
results = run_benchmark([source], tmp_path / "bench", baselines=["ours"])
assert results[0].metrics["document_reconstruction_cer"] == 0
assert results[0].metrics["document_reconstruction_wer"] == 0
assert results[0].metrics["document_reconstruction_similarity"] == 1.0
def test_structured_reference_table_scores_rowspan_colspan(tmp_path: Path) -> None:
source = tmp_path / "merged.png"
source.write_bytes(b"fake image bytes")
reference = source.with_suffix(".ref.json")
reference.write_text(
json.dumps(
{
"tables": [
{
"cells": [
{"row": 0, "col": 0, "text": "नाम", "rowspan": 1, "colspan": 2},
{"row": 1, "col": 0, "text": "राम"},
{"row": 1, "col": 1, "text": "१२"},
]
}
]
},
ensure_ascii=False,
),
encoding="utf-8",
)
loaded = load_reference(source)
assert loaded is not None
table = Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(0, 0, 100, 40),
cells=[
TableCell(row=0, col=0, text="नाम"),
TableCell(row=1, col=0, text="राम"),
TableCell(row=1, col=1, text="१२"),
],
)
document = Document(
source_path=str(source),
pages=[Page(page_index=0, blocks=[Block(block_id="p0-b0", block_type="table", page_index=0, bbox=table.bbox, order=0, table_id=table.table_id)])],
tables=[table],
)
metrics = score_document(document, "", loaded)
assert metrics["table_cell_f1"] == 1.0
assert metrics["table_exact_match"] == 0.0
assert metrics["table_structure_similarity"] < 1.0
assert metrics["table_teds_structure"] < 1.0
assert metrics["table_teds_text"] < 1.0
def test_table_scoring_matches_best_predicted_table_not_first_table(tmp_path: Path) -> None:
source = tmp_path / "tables.png"
source.write_bytes(b"fake image bytes")
source.with_suffix(".ref.json").write_text(
json.dumps({"tables": [[["Name", "Value"], ["A", "1"]]]}),
encoding="utf-8",
)
loaded = load_reference(source)
assert loaded is not None
decoy = Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(0, 0, 20, 20),
cells=[TableCell(row=0, col=0, text="wrong")],
)
correct = Table(
table_id="p0-t1",
page_index=0,
bbox=BBox(0, 30, 100, 40),
cells=[
TableCell(row=0, col=0, text="Name"),
TableCell(row=0, col=1, text="Value"),
TableCell(row=1, col=0, text="A"),
TableCell(row=1, col=1, text="1"),
],
)
document = Document(source_path=str(source), pages=[Page(page_index=0)], tables=[decoy, correct])
metrics = score_document(document, "", loaded)
assert metrics["table_cell_f1"] == 1.0
assert metrics["table_exact_match"] == 1.0
assert metrics["table_teds_structure"] == 1.0
assert metrics["table_teds_text"] == 1.0
def test_benchmark_scores_figure_caption_reference(tmp_path: Path) -> None:
source = tmp_path / "figure.txt"
source.write_text(
"वार्षिक प्रतिवेदन\n"
"[Figure: revenue chart]\n"
"चित्र १: Revenue growth from 2022 to 2026.\n"
"निष्कर्ष: राजस्व क्रमशः बढेको छ।\n",
encoding="utf-8",
)
source.with_suffix(".ref.json").write_text(
json.dumps(
{
"text": "वार्षिक प्रतिवेदन\nचित्र १: Revenue growth from 2022 to 2026.\nनिष्कर्ष: राजस्व क्रमशः बढेको छ।",
"reading_order": ["p0-l0", "p0-l1", "p0-l2", "p0-l3"],
"figures": [
{
"figure_id": "fig-revenue-chart",
"page_index": 0,
"caption": "चित्र १: Revenue growth from 2022 to 2026.",
"line_ids": ["p0-l1", "p0-l2"],
}
],
},
ensure_ascii=False,
),
encoding="utf-8",
)
results = run_benchmark([source], tmp_path / "bench", baselines=["ours"])
assert results[0].metrics["figure_detection_f1"] == 1.0
assert results[0].metrics["figure_caption_cer"] == 0
def test_benchmark_uses_manifest_reference_path_without_references_dir(tmp_path: Path) -> None:
source = tmp_path / "line.txt"
source.write_text("नेपाल सरकार\n", encoding="utf-8")
reference = tmp_path / "custom-reference.md"
reference.write_text("नेपाल सरकार\n", encoding="utf-8")
manifest = tmp_path / "eval.jsonl"
image_hash = sha256_file(source)
text_hash = sha256_text("नेपाल सरकार")
write_manifest(
[
ManifestEntry(
sample_id="sample-1",
dataset="fixture",
split="eval",
image_path=str(source),
text="नेपाल सरकार",
sha256=image_hash,
metadata={"reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")},
)
],
manifest,
)
results = run_benchmark([source], tmp_path / "bench", baselines=["ours"], eval_manifest=manifest)
assert results[0].metrics["cer"] == 0
assert results[0].metrics["wer"] == 0
def test_markdown_reference_path_scores_document_reconstruction(tmp_path: Path) -> None:
source = tmp_path / "title.txt"
source.write_text("नेपाल सरकार\n", encoding="utf-8")
reference = tmp_path / "document.md"
reference.write_text("# नेपाल सरकार\n", encoding="utf-8")
manifest = tmp_path / "eval.jsonl"
image_hash = sha256_file(source)
text_hash = sha256_text("# नेपाल सरकार")
write_manifest(
[
ManifestEntry(
sample_id="sample-md",
dataset="fixture",
split="eval",
image_path=str(source),
text="# नेपाल सरकार",
sha256=image_hash,
metadata={"reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")},
)
],
manifest,
)
results = run_benchmark([source], tmp_path / "bench", baselines=["ours"], eval_manifest=manifest)
assert results[0].metrics["document_reconstruction_similarity"] == 1.0
def test_materialize_references_writes_reference_pack_and_manifest(tmp_path: Path) -> None:
source = tmp_path / "sample.png"
source.write_bytes(b"png")
manifest = tmp_path / "eval.jsonl"
image_hash = sha256_file(source)
text_hash = sha256_text("hello world")
write_manifest(
[
ManifestEntry(
sample_id="sample-1",
dataset="fixture",
split="eval",
image_path=str(source),
text="hello world",
sha256=image_hash,
metadata={"text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")},
)
],
manifest,
)
report = materialize_references(manifest, tmp_path / "references", rewritten_manifest_path=tmp_path / "eval-with-refs.jsonl")
rewritten_rows = [json.loads(line) for line in (tmp_path / "eval-with-refs.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()]
assert Path(report.references_dir, "sample-1", "document.md").read_text(encoding="utf-8") == "hello world\n"
assert rewritten_rows[0]["metadata"]["reference_path"].endswith("sample-1/document.md")
assert Path(report.summary_json_path).exists()
def test_validate_claim_passes_configured_paired_gate(tmp_path: Path) -> None:
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
_row("ours", "s1", "a.png", ["nepali"], {"cer": 0.02}),
_row("tesseract", "s1", "a.png", ["nepali"], {"cer": 0.10}),
_row("ours", "s2", "b.png", ["nepali"], {"cer": 0.03}),
_row("tesseract", "s2", "b.png", ["nepali"], {"cer": 0.09}),
],
indent=2,
),
encoding="utf-8",
)
config = tmp_path / "config.json"
config.write_text(
json.dumps(
{
"candidate": "ours",
"min_samples": 2,
"required_slices": {"nepali": 2},
"bootstrap_iterations": 50,
"gates": [
{
"baseline": "tesseract",
"metric": "cer",
"direction": "lower",
"slice": "nepali",
"min_absolute_margin": 0.01,
"min_pair_win_rate": 0.5,
}
],
}
),
encoding="utf-8",
)
candidate_model = tmp_path / "candidate-model.json"
fallback_model = tmp_path / "fallback-model.json"
candidate_model.write_text('{"model_id":"candidate"}\n', encoding="utf-8")
fallback_model.write_text('{"model_id":"fallback"}\n', encoding="utf-8")
validation = validate_claim(
report,
tmp_path / "validation",
config_path=config,
candidate_model_config=candidate_model,
fallback_model_config=fallback_model,
)
assert validation.passed
assert validation.claim_status == "validated"
assert validation.gates[0].absolute_improvement
assert validation.gates[0].win_count == 2
assert validation.gates[0].loss_count == 0
assert validation.gates[0].tie_count == 0
assert validation.gates[0].win_rate == 1.0
assert validation.gates[0].pair_win_rate == 1.0
assert (tmp_path / "validation" / "validation.json").exists()
payload = json.loads((tmp_path / "validation" / "validation.json").read_text(encoding="utf-8"))
assert payload["gates"][0]["gate"]["min_pair_win_rate"] == 0.5
assert payload["gates"][0]["pair_win_rate"] == 1.0
assert payload["provenance"]["benchmark_report"]["path"] == str(report)
assert payload["provenance"]["benchmark_report"]["sha256"]
assert payload["provenance"]["validation_config"]["path"] == str(config)
assert payload["provenance"]["validation_config"]["sha256"]
assert payload["provenance"]["candidate_model_config"]["path"] == str(candidate_model)
assert payload["provenance"]["candidate_model_config"]["sha256"]
assert payload["provenance"]["fallback_model_config"]["path"] == str(fallback_model)
assert payload["provenance"]["fallback_model_config"]["sha256"]
def test_validate_claim_fails_with_missing_baseline_pairs(tmp_path: Path) -> None:
report = tmp_path / "report.json"
report.write_text(json.dumps([_row("ours", "s1", "a.png", ["nepali"], {"cer": 0.02})]), encoding="utf-8")
config = tmp_path / "config.json"
config.write_text(
json.dumps(
{
"candidate": "ours",
"min_samples": 1,
"gates": [{"baseline": "tesseract", "metric": "cer", "direction": "lower"}],
}
),
encoding="utf-8",
)
validation = validate_claim(report, tmp_path / "validation", config_path=config)
assert not validation.passed
assert "no paired samples" in validation.missing_requirements[0]
def test_validate_claim_can_require_pair_win_rate_over_all_pairs(tmp_path: Path) -> None:
rows = []
for index in range(1, 6):
metric = 0.09 if index == 1 else 0.10
rows.append(_row("ours", f"s{index}", f"{index}.png", ["nepali"], {"cer": metric}))
rows.append(_row("tesseract", f"s{index}", f"{index}.png", ["nepali"], {"cer": 0.10}))
report = tmp_path / "report.json"
report.write_text(json.dumps(rows, indent=2), encoding="utf-8")
config = tmp_path / "config.json"
config.write_text(
json.dumps(
{
"candidate": "ours",
"min_samples": 5,
"required_slices": {"nepali": 5},
"bootstrap_iterations": 0,
"gates": [
{
"baseline": "tesseract",
"metric": "cer",
"direction": "lower",
"slice": "nepali",
"min_pairs": 5,
"min_absolute_margin": 0.0,
"min_pair_win_rate": 0.5,
"require_ci_above_zero": False,
}
],
}
),
encoding="utf-8",
)
validation = validate_claim(report, tmp_path / "validation", config_path=config)
assert not validation.passed
assert validation.gates[0].win_count == 1
assert validation.gates[0].tie_count == 4
assert validation.gates[0].win_rate == 1.0
assert validation.gates[0].pair_win_rate == 0.2
assert "paired win rate 0.200000 < required 0.500000" in validation.missing_requirements[0]
def test_validate_claim_merges_eval_manifest_slices_when_report_already_has_slices(tmp_path: Path) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"image")
manifest = tmp_path / "hard-eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="s1",
dataset="fixture",
split="hard_eval",
image_path=str(source),
text="नेपाल",
metadata={"slices": ["hard_eval", "table"]},
)
],
manifest,
)
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
_row("ours", "s1", str(source), ["nepali"], {"cer": 0.01}),
_row("surya", "s1", str(source), ["nepali"], {"cer": 0.10}),
]
),
encoding="utf-8",
)
config = tmp_path / "config.json"
config.write_text(
json.dumps(
{
"candidate": "ours",
"min_samples": 1,
"required_slices": {"hard_eval": 1},
"bootstrap_iterations": 0,
"gates": [
{
"baseline": "surya",
"metric": "cer",
"direction": "lower",
"slice": "hard_eval",
"min_pairs": 1,
}
],
}
),
encoding="utf-8",
)
validation = validate_claim(report, tmp_path / "validation", config_path=config, eval_manifest=manifest)
assert validation.passed
assert validation.slice_counts["hard_eval"] == 1
assert validation.gates[0].pairs == 1
def test_validate_claim_can_require_full_eval_manifest_coverage(tmp_path: Path) -> None:
source_one = tmp_path / "page-1.png"
source_two = tmp_path / "page-2.png"
source_one.write_bytes(b"image one")
source_two.write_bytes(b"image two")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="s1",
dataset="fixture",
split="eval",
image_path=str(source_one),
text="नेपाल",
metadata={"slices": ["nepali"]},
),
ManifestEntry(
sample_id="s2",
dataset="fixture",
split="eval",
image_path=str(source_two),
text="सरकार",
metadata={"slices": ["nepali"]},
),
],
manifest,
)
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
_row("ours", "s1", str(source_one), ["nepali"], {"cer": 0.01}),
_row("surya", "s1", str(source_one), ["nepali"], {"cer": 0.10}),
]
),
encoding="utf-8",
)
config = tmp_path / "config.json"
config.write_text(
json.dumps(
{
"candidate": "ours",
"min_samples": 1,
"required_slices": {"nepali": 1},
"require_full_eval_manifest_coverage": True,
"bootstrap_iterations": 0,
"gates": [
{
"baseline": "surya",
"metric": "cer",
"direction": "lower",
"slice": "nepali",
"min_pairs": 1,
}
],
}
),
encoding="utf-8",
)
validation = validate_claim(report, tmp_path / "validation", config_path=config, eval_manifest=manifest)
assert not validation.passed
assert validation.gates[0].passed
assert "baseline 'ours' is missing successful rows for 1 eval manifest samples: s2" in validation.missing_requirements
assert "baseline 'surya' is missing successful rows for 1 eval manifest samples: s2" in validation.missing_requirements
def test_validate_claim_gate_can_require_multiple_slices(tmp_path: Path) -> None:
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
_row("ours", "hard-table", "a.png", ["hard_eval", "table"], {"cer": 0.01}),
_row("surya", "hard-table", "a.png", ["hard_eval", "table"], {"cer": 0.10}),
_row("ours", "other-table", "b.png", ["table"], {"cer": 0.90}),
_row("surya", "other-table", "b.png", ["table"], {"cer": 0.10}),
]
),
encoding="utf-8",
)
config = tmp_path / "config.json"
config.write_text(
json.dumps(
{
"candidate": "ours",
"min_samples": 2,
"bootstrap_iterations": 0,
"gates": [
{
"baseline": "surya",
"metric": "cer",
"direction": "lower",
"slices": ["hard_eval", "table"],
"min_pairs": 1,
}
],
}
),
encoding="utf-8",
)
validation = validate_claim(report, tmp_path / "validation", config_path=config)
assert validation.passed
assert validation.gates[0].pairs == 1
assert validation.gates[0].candidate_mean == 0.01
def test_validate_claim_supports_sign_test_gate(tmp_path: Path) -> None:
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
_row("ours", "s1", "a.png", ["nepali"], {"cer": 0.01}),
_row("tesseract", "s1", "a.png", ["nepali"], {"cer": 0.10}),
_row("ours", "s2", "b.png", ["nepali"], {"cer": 0.01}),
_row("tesseract", "s2", "b.png", ["nepali"], {"cer": 0.10}),
_row("ours", "s3", "c.png", ["nepali"], {"cer": 0.01}),
_row("tesseract", "s3", "c.png", ["nepali"], {"cer": 0.10}),
_row("ours", "s4", "d.png", ["nepali"], {"cer": 0.01}),
_row("tesseract", "s4", "d.png", ["nepali"], {"cer": 0.10}),
_row("ours", "s5", "e.png", ["nepali"], {"cer": 0.01}),
_row("tesseract", "s5", "e.png", ["nepali"], {"cer": 0.10}),
_row("ours", "s6", "f.png", ["nepali"], {"cer": 0.01}),
_row("tesseract", "s6", "f.png", ["nepali"], {"cer": 0.10}),
],
indent=2,
),
encoding="utf-8",
)
config = tmp_path / "config.json"
config.write_text(
json.dumps(
{
"candidate": "ours",
"min_samples": 6,
"required_slices": {"nepali": 6},
"bootstrap_iterations": 50,
"gates": [
{
"baseline": "tesseract",
"metric": "cer",
"direction": "lower",
"slice": "nepali",
"min_pairs": 6,
"max_p_value": 0.05,
}
],
}
),
encoding="utf-8",
)
validation = validate_claim(report, tmp_path / "validation", config_path=config)
assert validation.passed
assert validation.gates[0].sign_test_p_value is not None
assert validation.gates[0].sign_test_p_value <= 0.05
def test_validate_claim_reports_aggregate_metrics(tmp_path: Path) -> None:
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
_row("ours", "s1", "a.png", ["nepali"], {"cer": 0.02, "quality_score": 0.90}),
_row("ours", "s2", "b.png", ["english"], {"cer": 0.04, "quality_score": 0.80}),
_row("tesseract", "s1", "a.png", ["nepali"], {"cer": 0.10, "quality_score": 0.50}),
_row("tesseract", "s2", "b.png", ["english"], {"cer": 0.20, "quality_score": 0.40}),
],
indent=2,
),
encoding="utf-8",
)
config = tmp_path / "config.json"
config.write_text(
json.dumps(
{
"candidate": "ours",
"min_samples": 2,
"gates": [{"baseline": "tesseract", "metric": "cer", "direction": "lower", "min_pairs": 2}],
}
),
encoding="utf-8",
)
validation = validate_claim(report, tmp_path / "validation", config_path=config)
assert "ours" in validation.aggregate_metrics
assert validation.aggregate_metrics["ours"]["all"]["sample_count"] == 2
assert validation.aggregate_metrics["ours"]["nepali"]["cer"] == 0.02
def test_leakage_check_fails_on_image_hash_and_warns_on_text_only(tmp_path: Path) -> None:
shared_image = tmp_path / "shared.png"
eval_image = tmp_path / "eval.png"
shared_image.write_bytes(b"same")
eval_image.write_bytes(b"different")
image_hash = sha256_file(shared_image)
text_hash = sha256_text("same label")
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="train-1",
dataset="local",
split="train",
image_path=str(shared_image),
text="same label",
sha256=image_hash,
metadata={"text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")},
)
],
train_manifest,
)
write_manifest(
[
ManifestEntry(
sample_id="eval-1",
dataset="local",
split="eval",
image_path=str(shared_image),
text="same label",
sha256=image_hash,
metadata={"text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")},
),
ManifestEntry(
sample_id="eval-2",
dataset="local",
split="eval",
image_path=str(eval_image),
text="same label",
sha256=sha256_file(eval_image),
metadata={"text_sha256": text_hash},
),
],
eval_manifest,
)
report = check_leakage([train_manifest], eval_manifest)
assert not report.passed
assert image_hash in report.overlaps["image_sha256"]
assert text_hash in report.warnings["text_sha256"]
def _row(baseline: str, sample_id: str, input_path: str, slices: list[str], metrics: dict[str, float]) -> dict[str, object]:
return {
"baseline": baseline,
"input_path": input_path,
"status": "ok",
"latency_seconds": 1.0,
"sample_id": sample_id,
"slices": slices,
"metrics": metrics,
"error": None,
}