from __future__ import annotations import json from pathlib import Path from ocrtech.benchmark import run_benchmark from ocrtech.references import load_reference, score_document from ocrtech.manifest import ManifestEntry, sha256_file, sha256_text, write_manifest from ocrtech.references import materialize_references from ocrtech.schemas import BBox, Block, Document, Page, Table, TableCell from ocrtech.validation import check_leakage, validate_claim def test_benchmark_scores_reference_json_structure(tmp_path: Path) -> None: source = tmp_path / "page.txt" source.write_text( "नेपाल सरकार\n" "नाम उमेर जिल्ला\n" "राम ३२ काठमाडौं\n" "Sita 29 Lalitpur\n", encoding="utf-8", ) source.with_suffix(".ref.json").write_text( json.dumps( { "text": "नेपाल सरकार", "reading_order": ["p0-l0", "p0-l1", "p0-l2", "p0-l3"], "tables": [ [ ["नाम", "उमेर", "जिल्ला"], ["राम", "३२", "काठमाडौं"], ["Sita", "29", "Lalitpur"], ] ], }, ensure_ascii=False, ), encoding="utf-8", ) results = run_benchmark([source], tmp_path / "bench", baselines=["ours"]) assert results[0].metrics["reading_order_pair_accuracy"] == 1.0 assert results[0].metrics["table_cell_f1"] == 1.0 assert results[0].metrics["table_structure_similarity"] == 1.0 assert "text_cer" in results[0].metrics def test_benchmark_scores_document_reconstruction_markdown(tmp_path: Path) -> None: source = tmp_path / "page.txt" source.write_text( "नेपाल सरकार\n" "नाम उमेर जिल्ला\n" "राम ३२ काठमाडौं\n" "Sita 29 Lalitpur\n", encoding="utf-8", ) source.with_suffix(".ref.json").write_text( json.dumps( { "markdown": "# नेपाल सरकार\n\n| नाम | उमेर | जिल्ला |\n| --- | --- | --- |\n| राम | ३२ | काठमाडौं |\n| Sita | 29 | Lalitpur |\n", "tables": [ [ ["नाम", "उमेर", "जिल्ला"], ["राम", "३२", "काठमाडौं"], ["Sita", "29", "Lalitpur"], ] ], }, ensure_ascii=False, ), encoding="utf-8", ) results = run_benchmark([source], tmp_path / "bench", baselines=["ours"]) assert results[0].metrics["document_reconstruction_cer"] == 0 assert results[0].metrics["document_reconstruction_wer"] == 0 assert results[0].metrics["document_reconstruction_similarity"] == 1.0 def test_structured_reference_table_scores_rowspan_colspan(tmp_path: Path) -> None: source = tmp_path / "merged.png" source.write_bytes(b"fake image bytes") reference = source.with_suffix(".ref.json") reference.write_text( json.dumps( { "tables": [ { "cells": [ {"row": 0, "col": 0, "text": "नाम", "rowspan": 1, "colspan": 2}, {"row": 1, "col": 0, "text": "राम"}, {"row": 1, "col": 1, "text": "१२"}, ] } ] }, ensure_ascii=False, ), encoding="utf-8", ) loaded = load_reference(source) assert loaded is not None table = Table( table_id="p0-t0", page_index=0, bbox=BBox(0, 0, 100, 40), cells=[ TableCell(row=0, col=0, text="नाम"), TableCell(row=1, col=0, text="राम"), TableCell(row=1, col=1, text="१२"), ], ) document = Document( source_path=str(source), pages=[Page(page_index=0, blocks=[Block(block_id="p0-b0", block_type="table", page_index=0, bbox=table.bbox, order=0, table_id=table.table_id)])], tables=[table], ) metrics = score_document(document, "", loaded) assert metrics["table_cell_f1"] == 1.0 assert metrics["table_exact_match"] == 0.0 assert metrics["table_structure_similarity"] < 1.0 assert metrics["table_teds_structure"] < 1.0 assert metrics["table_teds_text"] < 1.0 def test_table_scoring_matches_best_predicted_table_not_first_table(tmp_path: Path) -> None: source = tmp_path / "tables.png" source.write_bytes(b"fake image bytes") source.with_suffix(".ref.json").write_text( json.dumps({"tables": [[["Name", "Value"], ["A", "1"]]]}), encoding="utf-8", ) loaded = load_reference(source) assert loaded is not None decoy = Table( table_id="p0-t0", page_index=0, bbox=BBox(0, 0, 20, 20), cells=[TableCell(row=0, col=0, text="wrong")], ) correct = Table( table_id="p0-t1", page_index=0, bbox=BBox(0, 30, 100, 40), cells=[ TableCell(row=0, col=0, text="Name"), TableCell(row=0, col=1, text="Value"), TableCell(row=1, col=0, text="A"), TableCell(row=1, col=1, text="1"), ], ) document = Document(source_path=str(source), pages=[Page(page_index=0)], tables=[decoy, correct]) metrics = score_document(document, "", loaded) assert metrics["table_cell_f1"] == 1.0 assert metrics["table_exact_match"] == 1.0 assert metrics["table_teds_structure"] == 1.0 assert metrics["table_teds_text"] == 1.0 def test_benchmark_scores_figure_caption_reference(tmp_path: Path) -> None: source = tmp_path / "figure.txt" source.write_text( "वार्षिक प्रतिवेदन\n" "[Figure: revenue chart]\n" "चित्र १: Revenue growth from 2022 to 2026.\n" "निष्कर्ष: राजस्व क्रमशः बढेको छ।\n", encoding="utf-8", ) source.with_suffix(".ref.json").write_text( json.dumps( { "text": "वार्षिक प्रतिवेदन\nचित्र १: Revenue growth from 2022 to 2026.\nनिष्कर्ष: राजस्व क्रमशः बढेको छ।", "reading_order": ["p0-l0", "p0-l1", "p0-l2", "p0-l3"], "figures": [ { "figure_id": "fig-revenue-chart", "page_index": 0, "caption": "चित्र १: Revenue growth from 2022 to 2026.", "line_ids": ["p0-l1", "p0-l2"], } ], }, ensure_ascii=False, ), encoding="utf-8", ) results = run_benchmark([source], tmp_path / "bench", baselines=["ours"]) assert results[0].metrics["figure_detection_f1"] == 1.0 assert results[0].metrics["figure_caption_cer"] == 0 def test_benchmark_uses_manifest_reference_path_without_references_dir(tmp_path: Path) -> None: source = tmp_path / "line.txt" source.write_text("नेपाल सरकार\n", encoding="utf-8") reference = tmp_path / "custom-reference.md" reference.write_text("नेपाल सरकार\n", encoding="utf-8") manifest = tmp_path / "eval.jsonl" image_hash = sha256_file(source) text_hash = sha256_text("नेपाल सरकार") write_manifest( [ ManifestEntry( sample_id="sample-1", dataset="fixture", split="eval", image_path=str(source), text="नेपाल सरकार", sha256=image_hash, metadata={"reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, ) ], manifest, ) results = run_benchmark([source], tmp_path / "bench", baselines=["ours"], eval_manifest=manifest) assert results[0].metrics["cer"] == 0 assert results[0].metrics["wer"] == 0 def test_markdown_reference_path_scores_document_reconstruction(tmp_path: Path) -> None: source = tmp_path / "title.txt" source.write_text("नेपाल सरकार\n", encoding="utf-8") reference = tmp_path / "document.md" reference.write_text("# नेपाल सरकार\n", encoding="utf-8") manifest = tmp_path / "eval.jsonl" image_hash = sha256_file(source) text_hash = sha256_text("# नेपाल सरकार") write_manifest( [ ManifestEntry( sample_id="sample-md", dataset="fixture", split="eval", image_path=str(source), text="# नेपाल सरकार", sha256=image_hash, metadata={"reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, ) ], manifest, ) results = run_benchmark([source], tmp_path / "bench", baselines=["ours"], eval_manifest=manifest) assert results[0].metrics["document_reconstruction_similarity"] == 1.0 def test_materialize_references_writes_reference_pack_and_manifest(tmp_path: Path) -> None: source = tmp_path / "sample.png" source.write_bytes(b"png") manifest = tmp_path / "eval.jsonl" image_hash = sha256_file(source) text_hash = sha256_text("hello world") write_manifest( [ ManifestEntry( sample_id="sample-1", dataset="fixture", split="eval", image_path=str(source), text="hello world", sha256=image_hash, metadata={"text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, ) ], manifest, ) report = materialize_references(manifest, tmp_path / "references", rewritten_manifest_path=tmp_path / "eval-with-refs.jsonl") rewritten_rows = [json.loads(line) for line in (tmp_path / "eval-with-refs.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()] assert Path(report.references_dir, "sample-1", "document.md").read_text(encoding="utf-8") == "hello world\n" assert rewritten_rows[0]["metadata"]["reference_path"].endswith("sample-1/document.md") assert Path(report.summary_json_path).exists() def test_validate_claim_passes_configured_paired_gate(tmp_path: Path) -> None: report = tmp_path / "report.json" report.write_text( json.dumps( [ _row("ours", "s1", "a.png", ["nepali"], {"cer": 0.02}), _row("tesseract", "s1", "a.png", ["nepali"], {"cer": 0.10}), _row("ours", "s2", "b.png", ["nepali"], {"cer": 0.03}), _row("tesseract", "s2", "b.png", ["nepali"], {"cer": 0.09}), ], indent=2, ), encoding="utf-8", ) config = tmp_path / "config.json" config.write_text( json.dumps( { "candidate": "ours", "min_samples": 2, "required_slices": {"nepali": 2}, "bootstrap_iterations": 50, "gates": [ { "baseline": "tesseract", "metric": "cer", "direction": "lower", "slice": "nepali", "min_absolute_margin": 0.01, "min_pair_win_rate": 0.5, } ], } ), encoding="utf-8", ) candidate_model = tmp_path / "candidate-model.json" fallback_model = tmp_path / "fallback-model.json" candidate_model.write_text('{"model_id":"candidate"}\n', encoding="utf-8") fallback_model.write_text('{"model_id":"fallback"}\n', encoding="utf-8") validation = validate_claim( report, tmp_path / "validation", config_path=config, candidate_model_config=candidate_model, fallback_model_config=fallback_model, ) assert validation.passed assert validation.claim_status == "validated" assert validation.gates[0].absolute_improvement assert validation.gates[0].win_count == 2 assert validation.gates[0].loss_count == 0 assert validation.gates[0].tie_count == 0 assert validation.gates[0].win_rate == 1.0 assert validation.gates[0].pair_win_rate == 1.0 assert (tmp_path / "validation" / "validation.json").exists() payload = json.loads((tmp_path / "validation" / "validation.json").read_text(encoding="utf-8")) assert payload["gates"][0]["gate"]["min_pair_win_rate"] == 0.5 assert payload["gates"][0]["pair_win_rate"] == 1.0 assert payload["provenance"]["benchmark_report"]["path"] == str(report) assert payload["provenance"]["benchmark_report"]["sha256"] assert payload["provenance"]["validation_config"]["path"] == str(config) assert payload["provenance"]["validation_config"]["sha256"] assert payload["provenance"]["candidate_model_config"]["path"] == str(candidate_model) assert payload["provenance"]["candidate_model_config"]["sha256"] assert payload["provenance"]["fallback_model_config"]["path"] == str(fallback_model) assert payload["provenance"]["fallback_model_config"]["sha256"] def test_validate_claim_fails_with_missing_baseline_pairs(tmp_path: Path) -> None: report = tmp_path / "report.json" report.write_text(json.dumps([_row("ours", "s1", "a.png", ["nepali"], {"cer": 0.02})]), encoding="utf-8") config = tmp_path / "config.json" config.write_text( json.dumps( { "candidate": "ours", "min_samples": 1, "gates": [{"baseline": "tesseract", "metric": "cer", "direction": "lower"}], } ), encoding="utf-8", ) validation = validate_claim(report, tmp_path / "validation", config_path=config) assert not validation.passed assert "no paired samples" in validation.missing_requirements[0] def test_validate_claim_can_require_pair_win_rate_over_all_pairs(tmp_path: Path) -> None: rows = [] for index in range(1, 6): metric = 0.09 if index == 1 else 0.10 rows.append(_row("ours", f"s{index}", f"{index}.png", ["nepali"], {"cer": metric})) rows.append(_row("tesseract", f"s{index}", f"{index}.png", ["nepali"], {"cer": 0.10})) report = tmp_path / "report.json" report.write_text(json.dumps(rows, indent=2), encoding="utf-8") config = tmp_path / "config.json" config.write_text( json.dumps( { "candidate": "ours", "min_samples": 5, "required_slices": {"nepali": 5}, "bootstrap_iterations": 0, "gates": [ { "baseline": "tesseract", "metric": "cer", "direction": "lower", "slice": "nepali", "min_pairs": 5, "min_absolute_margin": 0.0, "min_pair_win_rate": 0.5, "require_ci_above_zero": False, } ], } ), encoding="utf-8", ) validation = validate_claim(report, tmp_path / "validation", config_path=config) assert not validation.passed assert validation.gates[0].win_count == 1 assert validation.gates[0].tie_count == 4 assert validation.gates[0].win_rate == 1.0 assert validation.gates[0].pair_win_rate == 0.2 assert "paired win rate 0.200000 < required 0.500000" in validation.missing_requirements[0] def test_validate_claim_merges_eval_manifest_slices_when_report_already_has_slices(tmp_path: Path) -> None: source = tmp_path / "page.png" source.write_bytes(b"image") manifest = tmp_path / "hard-eval.jsonl" write_manifest( [ ManifestEntry( sample_id="s1", dataset="fixture", split="hard_eval", image_path=str(source), text="नेपाल", metadata={"slices": ["hard_eval", "table"]}, ) ], manifest, ) report = tmp_path / "report.json" report.write_text( json.dumps( [ _row("ours", "s1", str(source), ["nepali"], {"cer": 0.01}), _row("surya", "s1", str(source), ["nepali"], {"cer": 0.10}), ] ), encoding="utf-8", ) config = tmp_path / "config.json" config.write_text( json.dumps( { "candidate": "ours", "min_samples": 1, "required_slices": {"hard_eval": 1}, "bootstrap_iterations": 0, "gates": [ { "baseline": "surya", "metric": "cer", "direction": "lower", "slice": "hard_eval", "min_pairs": 1, } ], } ), encoding="utf-8", ) validation = validate_claim(report, tmp_path / "validation", config_path=config, eval_manifest=manifest) assert validation.passed assert validation.slice_counts["hard_eval"] == 1 assert validation.gates[0].pairs == 1 def test_validate_claim_can_require_full_eval_manifest_coverage(tmp_path: Path) -> None: source_one = tmp_path / "page-1.png" source_two = tmp_path / "page-2.png" source_one.write_bytes(b"image one") source_two.write_bytes(b"image two") manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="s1", dataset="fixture", split="eval", image_path=str(source_one), text="नेपाल", metadata={"slices": ["nepali"]}, ), ManifestEntry( sample_id="s2", dataset="fixture", split="eval", image_path=str(source_two), text="सरकार", metadata={"slices": ["nepali"]}, ), ], manifest, ) report = tmp_path / "report.json" report.write_text( json.dumps( [ _row("ours", "s1", str(source_one), ["nepali"], {"cer": 0.01}), _row("surya", "s1", str(source_one), ["nepali"], {"cer": 0.10}), ] ), encoding="utf-8", ) config = tmp_path / "config.json" config.write_text( json.dumps( { "candidate": "ours", "min_samples": 1, "required_slices": {"nepali": 1}, "require_full_eval_manifest_coverage": True, "bootstrap_iterations": 0, "gates": [ { "baseline": "surya", "metric": "cer", "direction": "lower", "slice": "nepali", "min_pairs": 1, } ], } ), encoding="utf-8", ) validation = validate_claim(report, tmp_path / "validation", config_path=config, eval_manifest=manifest) assert not validation.passed assert validation.gates[0].passed assert "baseline 'ours' is missing successful rows for 1 eval manifest samples: s2" in validation.missing_requirements assert "baseline 'surya' is missing successful rows for 1 eval manifest samples: s2" in validation.missing_requirements def test_validate_claim_gate_can_require_multiple_slices(tmp_path: Path) -> None: report = tmp_path / "report.json" report.write_text( json.dumps( [ _row("ours", "hard-table", "a.png", ["hard_eval", "table"], {"cer": 0.01}), _row("surya", "hard-table", "a.png", ["hard_eval", "table"], {"cer": 0.10}), _row("ours", "other-table", "b.png", ["table"], {"cer": 0.90}), _row("surya", "other-table", "b.png", ["table"], {"cer": 0.10}), ] ), encoding="utf-8", ) config = tmp_path / "config.json" config.write_text( json.dumps( { "candidate": "ours", "min_samples": 2, "bootstrap_iterations": 0, "gates": [ { "baseline": "surya", "metric": "cer", "direction": "lower", "slices": ["hard_eval", "table"], "min_pairs": 1, } ], } ), encoding="utf-8", ) validation = validate_claim(report, tmp_path / "validation", config_path=config) assert validation.passed assert validation.gates[0].pairs == 1 assert validation.gates[0].candidate_mean == 0.01 def test_validate_claim_supports_sign_test_gate(tmp_path: Path) -> None: report = tmp_path / "report.json" report.write_text( json.dumps( [ _row("ours", "s1", "a.png", ["nepali"], {"cer": 0.01}), _row("tesseract", "s1", "a.png", ["nepali"], {"cer": 0.10}), _row("ours", "s2", "b.png", ["nepali"], {"cer": 0.01}), _row("tesseract", "s2", "b.png", ["nepali"], {"cer": 0.10}), _row("ours", "s3", "c.png", ["nepali"], {"cer": 0.01}), _row("tesseract", "s3", "c.png", ["nepali"], {"cer": 0.10}), _row("ours", "s4", "d.png", ["nepali"], {"cer": 0.01}), _row("tesseract", "s4", "d.png", ["nepali"], {"cer": 0.10}), _row("ours", "s5", "e.png", ["nepali"], {"cer": 0.01}), _row("tesseract", "s5", "e.png", ["nepali"], {"cer": 0.10}), _row("ours", "s6", "f.png", ["nepali"], {"cer": 0.01}), _row("tesseract", "s6", "f.png", ["nepali"], {"cer": 0.10}), ], indent=2, ), encoding="utf-8", ) config = tmp_path / "config.json" config.write_text( json.dumps( { "candidate": "ours", "min_samples": 6, "required_slices": {"nepali": 6}, "bootstrap_iterations": 50, "gates": [ { "baseline": "tesseract", "metric": "cer", "direction": "lower", "slice": "nepali", "min_pairs": 6, "max_p_value": 0.05, } ], } ), encoding="utf-8", ) validation = validate_claim(report, tmp_path / "validation", config_path=config) assert validation.passed assert validation.gates[0].sign_test_p_value is not None assert validation.gates[0].sign_test_p_value <= 0.05 def test_validate_claim_reports_aggregate_metrics(tmp_path: Path) -> None: report = tmp_path / "report.json" report.write_text( json.dumps( [ _row("ours", "s1", "a.png", ["nepali"], {"cer": 0.02, "quality_score": 0.90}), _row("ours", "s2", "b.png", ["english"], {"cer": 0.04, "quality_score": 0.80}), _row("tesseract", "s1", "a.png", ["nepali"], {"cer": 0.10, "quality_score": 0.50}), _row("tesseract", "s2", "b.png", ["english"], {"cer": 0.20, "quality_score": 0.40}), ], indent=2, ), encoding="utf-8", ) config = tmp_path / "config.json" config.write_text( json.dumps( { "candidate": "ours", "min_samples": 2, "gates": [{"baseline": "tesseract", "metric": "cer", "direction": "lower", "min_pairs": 2}], } ), encoding="utf-8", ) validation = validate_claim(report, tmp_path / "validation", config_path=config) assert "ours" in validation.aggregate_metrics assert validation.aggregate_metrics["ours"]["all"]["sample_count"] == 2 assert validation.aggregate_metrics["ours"]["nepali"]["cer"] == 0.02 def test_leakage_check_fails_on_image_hash_and_warns_on_text_only(tmp_path: Path) -> None: shared_image = tmp_path / "shared.png" eval_image = tmp_path / "eval.png" shared_image.write_bytes(b"same") eval_image.write_bytes(b"different") image_hash = sha256_file(shared_image) text_hash = sha256_text("same label") train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="train-1", dataset="local", split="train", image_path=str(shared_image), text="same label", sha256=image_hash, metadata={"text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, ) ], train_manifest, ) write_manifest( [ ManifestEntry( sample_id="eval-1", dataset="local", split="eval", image_path=str(shared_image), text="same label", sha256=image_hash, metadata={"text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, ), ManifestEntry( sample_id="eval-2", dataset="local", split="eval", image_path=str(eval_image), text="same label", sha256=sha256_file(eval_image), metadata={"text_sha256": text_hash}, ), ], eval_manifest, ) report = check_leakage([train_manifest], eval_manifest) assert not report.passed assert image_hash in report.overlaps["image_sha256"] assert text_hash in report.warnings["text_sha256"] def _row(baseline: str, sample_id: str, input_path: str, slices: list[str], metrics: dict[str, float]) -> dict[str, object]: return { "baseline": baseline, "input_path": input_path, "status": "ok", "latency_seconds": 1.0, "sample_id": sample_id, "slices": slices, "metrics": metrics, "error": None, }