Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_references_validation.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 27.3 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_references_validation.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_references_validation.py
-
curl -L -o test_references_validation.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_references_validation.py
27.3 kB
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| from ocrtech.benchmark import run_benchmark | |
| from ocrtech.references import load_reference, score_document | |
| from ocrtech.manifest import ManifestEntry, sha256_file, sha256_text, write_manifest | |
| from ocrtech.references import materialize_references | |
| from ocrtech.schemas import BBox, Block, Document, Page, Table, TableCell | |
| from ocrtech.validation import check_leakage, validate_claim | |
| def test_benchmark_scores_reference_json_structure(tmp_path: Path) -> None: | |
| source = tmp_path / "page.txt" | |
| source.write_text( | |
| "नेपाल सरकार\n" | |
| "नाम उमेर जिल्ला\n" | |
| "राम ३२ काठमाडौं\n" | |
| "Sita 29 Lalitpur\n", | |
| encoding="utf-8", | |
| ) | |
| source.with_suffix(".ref.json").write_text( | |
| json.dumps( | |
| { | |
| "text": "नेपाल सरकार", | |
| "reading_order": ["p0-l0", "p0-l1", "p0-l2", "p0-l3"], | |
| "tables": [ | |
| [ | |
| ["नाम", "उमेर", "जिल्ला"], | |
| ["राम", "३२", "काठमाडौं"], | |
| ["Sita", "29", "Lalitpur"], | |
| ] | |
| ], | |
| }, | |
| ensure_ascii=False, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| results = run_benchmark([source], tmp_path / "bench", baselines=["ours"]) | |
| assert results[0].metrics["reading_order_pair_accuracy"] == 1.0 | |
| assert results[0].metrics["table_cell_f1"] == 1.0 | |
| assert results[0].metrics["table_structure_similarity"] == 1.0 | |
| assert "text_cer" in results[0].metrics | |
| def test_benchmark_scores_document_reconstruction_markdown(tmp_path: Path) -> None: | |
| source = tmp_path / "page.txt" | |
| source.write_text( | |
| "नेपाल सरकार\n" | |
| "नाम उमेर जिल्ला\n" | |
| "राम ३२ काठमाडौं\n" | |
| "Sita 29 Lalitpur\n", | |
| encoding="utf-8", | |
| ) | |
| source.with_suffix(".ref.json").write_text( | |
| json.dumps( | |
| { | |
| "markdown": "# नेपाल सरकार\n\n| नाम | उमेर | जिल्ला |\n| --- | --- | --- |\n| राम | ३२ | काठमाडौं |\n| Sita | 29 | Lalitpur |\n", | |
| "tables": [ | |
| [ | |
| ["नाम", "उमेर", "जिल्ला"], | |
| ["राम", "३२", "काठमाडौं"], | |
| ["Sita", "29", "Lalitpur"], | |
| ] | |
| ], | |
| }, | |
| ensure_ascii=False, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| results = run_benchmark([source], tmp_path / "bench", baselines=["ours"]) | |
| assert results[0].metrics["document_reconstruction_cer"] == 0 | |
| assert results[0].metrics["document_reconstruction_wer"] == 0 | |
| assert results[0].metrics["document_reconstruction_similarity"] == 1.0 | |
| def test_structured_reference_table_scores_rowspan_colspan(tmp_path: Path) -> None: | |
| source = tmp_path / "merged.png" | |
| source.write_bytes(b"fake image bytes") | |
| reference = source.with_suffix(".ref.json") | |
| reference.write_text( | |
| json.dumps( | |
| { | |
| "tables": [ | |
| { | |
| "cells": [ | |
| {"row": 0, "col": 0, "text": "नाम", "rowspan": 1, "colspan": 2}, | |
| {"row": 1, "col": 0, "text": "राम"}, | |
| {"row": 1, "col": 1, "text": "१२"}, | |
| ] | |
| } | |
| ] | |
| }, | |
| ensure_ascii=False, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| loaded = load_reference(source) | |
| assert loaded is not None | |
| table = Table( | |
| table_id="p0-t0", | |
| page_index=0, | |
| bbox=BBox(0, 0, 100, 40), | |
| cells=[ | |
| TableCell(row=0, col=0, text="नाम"), | |
| TableCell(row=1, col=0, text="राम"), | |
| TableCell(row=1, col=1, text="१२"), | |
| ], | |
| ) | |
| document = Document( | |
| source_path=str(source), | |
| pages=[Page(page_index=0, blocks=[Block(block_id="p0-b0", block_type="table", page_index=0, bbox=table.bbox, order=0, table_id=table.table_id)])], | |
| tables=[table], | |
| ) | |
| metrics = score_document(document, "", loaded) | |
| assert metrics["table_cell_f1"] == 1.0 | |
| assert metrics["table_exact_match"] == 0.0 | |
| assert metrics["table_structure_similarity"] < 1.0 | |
| assert metrics["table_teds_structure"] < 1.0 | |
| assert metrics["table_teds_text"] < 1.0 | |
| def test_table_scoring_matches_best_predicted_table_not_first_table(tmp_path: Path) -> None: | |
| source = tmp_path / "tables.png" | |
| source.write_bytes(b"fake image bytes") | |
| source.with_suffix(".ref.json").write_text( | |
| json.dumps({"tables": [[["Name", "Value"], ["A", "1"]]]}), | |
| encoding="utf-8", | |
| ) | |
| loaded = load_reference(source) | |
| assert loaded is not None | |
| decoy = Table( | |
| table_id="p0-t0", | |
| page_index=0, | |
| bbox=BBox(0, 0, 20, 20), | |
| cells=[TableCell(row=0, col=0, text="wrong")], | |
| ) | |
| correct = Table( | |
| table_id="p0-t1", | |
| page_index=0, | |
| bbox=BBox(0, 30, 100, 40), | |
| cells=[ | |
| TableCell(row=0, col=0, text="Name"), | |
| TableCell(row=0, col=1, text="Value"), | |
| TableCell(row=1, col=0, text="A"), | |
| TableCell(row=1, col=1, text="1"), | |
| ], | |
| ) | |
| document = Document(source_path=str(source), pages=[Page(page_index=0)], tables=[decoy, correct]) | |
| metrics = score_document(document, "", loaded) | |
| assert metrics["table_cell_f1"] == 1.0 | |
| assert metrics["table_exact_match"] == 1.0 | |
| assert metrics["table_teds_structure"] == 1.0 | |
| assert metrics["table_teds_text"] == 1.0 | |
| def test_benchmark_scores_figure_caption_reference(tmp_path: Path) -> None: | |
| source = tmp_path / "figure.txt" | |
| source.write_text( | |
| "वार्षिक प्रतिवेदन\n" | |
| "[Figure: revenue chart]\n" | |
| "चित्र १: Revenue growth from 2022 to 2026.\n" | |
| "निष्कर्ष: राजस्व क्रमशः बढेको छ।\n", | |
| encoding="utf-8", | |
| ) | |
| source.with_suffix(".ref.json").write_text( | |
| json.dumps( | |
| { | |
| "text": "वार्षिक प्रतिवेदन\nचित्र १: Revenue growth from 2022 to 2026.\nनिष्कर्ष: राजस्व क्रमशः बढेको छ।", | |
| "reading_order": ["p0-l0", "p0-l1", "p0-l2", "p0-l3"], | |
| "figures": [ | |
| { | |
| "figure_id": "fig-revenue-chart", | |
| "page_index": 0, | |
| "caption": "चित्र १: Revenue growth from 2022 to 2026.", | |
| "line_ids": ["p0-l1", "p0-l2"], | |
| } | |
| ], | |
| }, | |
| ensure_ascii=False, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| results = run_benchmark([source], tmp_path / "bench", baselines=["ours"]) | |
| assert results[0].metrics["figure_detection_f1"] == 1.0 | |
| assert results[0].metrics["figure_caption_cer"] == 0 | |
| def test_benchmark_uses_manifest_reference_path_without_references_dir(tmp_path: Path) -> None: | |
| source = tmp_path / "line.txt" | |
| source.write_text("नेपाल सरकार\n", encoding="utf-8") | |
| reference = tmp_path / "custom-reference.md" | |
| reference.write_text("नेपाल सरकार\n", encoding="utf-8") | |
| manifest = tmp_path / "eval.jsonl" | |
| image_hash = sha256_file(source) | |
| text_hash = sha256_text("नेपाल सरकार") | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="sample-1", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(source), | |
| text="नेपाल सरकार", | |
| sha256=image_hash, | |
| metadata={"reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| results = run_benchmark([source], tmp_path / "bench", baselines=["ours"], eval_manifest=manifest) | |
| assert results[0].metrics["cer"] == 0 | |
| assert results[0].metrics["wer"] == 0 | |
| def test_markdown_reference_path_scores_document_reconstruction(tmp_path: Path) -> None: | |
| source = tmp_path / "title.txt" | |
| source.write_text("नेपाल सरकार\n", encoding="utf-8") | |
| reference = tmp_path / "document.md" | |
| reference.write_text("# नेपाल सरकार\n", encoding="utf-8") | |
| manifest = tmp_path / "eval.jsonl" | |
| image_hash = sha256_file(source) | |
| text_hash = sha256_text("# नेपाल सरकार") | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="sample-md", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(source), | |
| text="# नेपाल सरकार", | |
| sha256=image_hash, | |
| metadata={"reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| results = run_benchmark([source], tmp_path / "bench", baselines=["ours"], eval_manifest=manifest) | |
| assert results[0].metrics["document_reconstruction_similarity"] == 1.0 | |
| def test_materialize_references_writes_reference_pack_and_manifest(tmp_path: Path) -> None: | |
| source = tmp_path / "sample.png" | |
| source.write_bytes(b"png") | |
| manifest = tmp_path / "eval.jsonl" | |
| image_hash = sha256_file(source) | |
| text_hash = sha256_text("hello world") | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="sample-1", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(source), | |
| text="hello world", | |
| sha256=image_hash, | |
| metadata={"text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| report = materialize_references(manifest, tmp_path / "references", rewritten_manifest_path=tmp_path / "eval-with-refs.jsonl") | |
| rewritten_rows = [json.loads(line) for line in (tmp_path / "eval-with-refs.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()] | |
| assert Path(report.references_dir, "sample-1", "document.md").read_text(encoding="utf-8") == "hello world\n" | |
| assert rewritten_rows[0]["metadata"]["reference_path"].endswith("sample-1/document.md") | |
| assert Path(report.summary_json_path).exists() | |
| def test_validate_claim_passes_configured_paired_gate(tmp_path: Path) -> None: | |
| report = tmp_path / "report.json" | |
| report.write_text( | |
| json.dumps( | |
| [ | |
| _row("ours", "s1", "a.png", ["nepali"], {"cer": 0.02}), | |
| _row("tesseract", "s1", "a.png", ["nepali"], {"cer": 0.10}), | |
| _row("ours", "s2", "b.png", ["nepali"], {"cer": 0.03}), | |
| _row("tesseract", "s2", "b.png", ["nepali"], {"cer": 0.09}), | |
| ], | |
| indent=2, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| config = tmp_path / "config.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "ours", | |
| "min_samples": 2, | |
| "required_slices": {"nepali": 2}, | |
| "bootstrap_iterations": 50, | |
| "gates": [ | |
| { | |
| "baseline": "tesseract", | |
| "metric": "cer", | |
| "direction": "lower", | |
| "slice": "nepali", | |
| "min_absolute_margin": 0.01, | |
| "min_pair_win_rate": 0.5, | |
| } | |
| ], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| candidate_model = tmp_path / "candidate-model.json" | |
| fallback_model = tmp_path / "fallback-model.json" | |
| candidate_model.write_text('{"model_id":"candidate"}\n', encoding="utf-8") | |
| fallback_model.write_text('{"model_id":"fallback"}\n', encoding="utf-8") | |
| validation = validate_claim( | |
| report, | |
| tmp_path / "validation", | |
| config_path=config, | |
| candidate_model_config=candidate_model, | |
| fallback_model_config=fallback_model, | |
| ) | |
| assert validation.passed | |
| assert validation.claim_status == "validated" | |
| assert validation.gates[0].absolute_improvement | |
| assert validation.gates[0].win_count == 2 | |
| assert validation.gates[0].loss_count == 0 | |
| assert validation.gates[0].tie_count == 0 | |
| assert validation.gates[0].win_rate == 1.0 | |
| assert validation.gates[0].pair_win_rate == 1.0 | |
| assert (tmp_path / "validation" / "validation.json").exists() | |
| payload = json.loads((tmp_path / "validation" / "validation.json").read_text(encoding="utf-8")) | |
| assert payload["gates"][0]["gate"]["min_pair_win_rate"] == 0.5 | |
| assert payload["gates"][0]["pair_win_rate"] == 1.0 | |
| assert payload["provenance"]["benchmark_report"]["path"] == str(report) | |
| assert payload["provenance"]["benchmark_report"]["sha256"] | |
| assert payload["provenance"]["validation_config"]["path"] == str(config) | |
| assert payload["provenance"]["validation_config"]["sha256"] | |
| assert payload["provenance"]["candidate_model_config"]["path"] == str(candidate_model) | |
| assert payload["provenance"]["candidate_model_config"]["sha256"] | |
| assert payload["provenance"]["fallback_model_config"]["path"] == str(fallback_model) | |
| assert payload["provenance"]["fallback_model_config"]["sha256"] | |
| def test_validate_claim_fails_with_missing_baseline_pairs(tmp_path: Path) -> None: | |
| report = tmp_path / "report.json" | |
| report.write_text(json.dumps([_row("ours", "s1", "a.png", ["nepali"], {"cer": 0.02})]), encoding="utf-8") | |
| config = tmp_path / "config.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "ours", | |
| "min_samples": 1, | |
| "gates": [{"baseline": "tesseract", "metric": "cer", "direction": "lower"}], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| validation = validate_claim(report, tmp_path / "validation", config_path=config) | |
| assert not validation.passed | |
| assert "no paired samples" in validation.missing_requirements[0] | |
| def test_validate_claim_can_require_pair_win_rate_over_all_pairs(tmp_path: Path) -> None: | |
| rows = [] | |
| for index in range(1, 6): | |
| metric = 0.09 if index == 1 else 0.10 | |
| rows.append(_row("ours", f"s{index}", f"{index}.png", ["nepali"], {"cer": metric})) | |
| rows.append(_row("tesseract", f"s{index}", f"{index}.png", ["nepali"], {"cer": 0.10})) | |
| report = tmp_path / "report.json" | |
| report.write_text(json.dumps(rows, indent=2), encoding="utf-8") | |
| config = tmp_path / "config.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "ours", | |
| "min_samples": 5, | |
| "required_slices": {"nepali": 5}, | |
| "bootstrap_iterations": 0, | |
| "gates": [ | |
| { | |
| "baseline": "tesseract", | |
| "metric": "cer", | |
| "direction": "lower", | |
| "slice": "nepali", | |
| "min_pairs": 5, | |
| "min_absolute_margin": 0.0, | |
| "min_pair_win_rate": 0.5, | |
| "require_ci_above_zero": False, | |
| } | |
| ], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| validation = validate_claim(report, tmp_path / "validation", config_path=config) | |
| assert not validation.passed | |
| assert validation.gates[0].win_count == 1 | |
| assert validation.gates[0].tie_count == 4 | |
| assert validation.gates[0].win_rate == 1.0 | |
| assert validation.gates[0].pair_win_rate == 0.2 | |
| assert "paired win rate 0.200000 < required 0.500000" in validation.missing_requirements[0] | |
| def test_validate_claim_merges_eval_manifest_slices_when_report_already_has_slices(tmp_path: Path) -> None: | |
| source = tmp_path / "page.png" | |
| source.write_bytes(b"image") | |
| manifest = tmp_path / "hard-eval.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="s1", | |
| dataset="fixture", | |
| split="hard_eval", | |
| image_path=str(source), | |
| text="नेपाल", | |
| metadata={"slices": ["hard_eval", "table"]}, | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| report = tmp_path / "report.json" | |
| report.write_text( | |
| json.dumps( | |
| [ | |
| _row("ours", "s1", str(source), ["nepali"], {"cer": 0.01}), | |
| _row("surya", "s1", str(source), ["nepali"], {"cer": 0.10}), | |
| ] | |
| ), | |
| encoding="utf-8", | |
| ) | |
| config = tmp_path / "config.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "ours", | |
| "min_samples": 1, | |
| "required_slices": {"hard_eval": 1}, | |
| "bootstrap_iterations": 0, | |
| "gates": [ | |
| { | |
| "baseline": "surya", | |
| "metric": "cer", | |
| "direction": "lower", | |
| "slice": "hard_eval", | |
| "min_pairs": 1, | |
| } | |
| ], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| validation = validate_claim(report, tmp_path / "validation", config_path=config, eval_manifest=manifest) | |
| assert validation.passed | |
| assert validation.slice_counts["hard_eval"] == 1 | |
| assert validation.gates[0].pairs == 1 | |
| def test_validate_claim_can_require_full_eval_manifest_coverage(tmp_path: Path) -> None: | |
| source_one = tmp_path / "page-1.png" | |
| source_two = tmp_path / "page-2.png" | |
| source_one.write_bytes(b"image one") | |
| source_two.write_bytes(b"image two") | |
| manifest = tmp_path / "eval.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="s1", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(source_one), | |
| text="नेपाल", | |
| metadata={"slices": ["nepali"]}, | |
| ), | |
| ManifestEntry( | |
| sample_id="s2", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(source_two), | |
| text="सरकार", | |
| metadata={"slices": ["nepali"]}, | |
| ), | |
| ], | |
| manifest, | |
| ) | |
| report = tmp_path / "report.json" | |
| report.write_text( | |
| json.dumps( | |
| [ | |
| _row("ours", "s1", str(source_one), ["nepali"], {"cer": 0.01}), | |
| _row("surya", "s1", str(source_one), ["nepali"], {"cer": 0.10}), | |
| ] | |
| ), | |
| encoding="utf-8", | |
| ) | |
| config = tmp_path / "config.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "ours", | |
| "min_samples": 1, | |
| "required_slices": {"nepali": 1}, | |
| "require_full_eval_manifest_coverage": True, | |
| "bootstrap_iterations": 0, | |
| "gates": [ | |
| { | |
| "baseline": "surya", | |
| "metric": "cer", | |
| "direction": "lower", | |
| "slice": "nepali", | |
| "min_pairs": 1, | |
| } | |
| ], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| validation = validate_claim(report, tmp_path / "validation", config_path=config, eval_manifest=manifest) | |
| assert not validation.passed | |
| assert validation.gates[0].passed | |
| assert "baseline 'ours' is missing successful rows for 1 eval manifest samples: s2" in validation.missing_requirements | |
| assert "baseline 'surya' is missing successful rows for 1 eval manifest samples: s2" in validation.missing_requirements | |
| def test_validate_claim_gate_can_require_multiple_slices(tmp_path: Path) -> None: | |
| report = tmp_path / "report.json" | |
| report.write_text( | |
| json.dumps( | |
| [ | |
| _row("ours", "hard-table", "a.png", ["hard_eval", "table"], {"cer": 0.01}), | |
| _row("surya", "hard-table", "a.png", ["hard_eval", "table"], {"cer": 0.10}), | |
| _row("ours", "other-table", "b.png", ["table"], {"cer": 0.90}), | |
| _row("surya", "other-table", "b.png", ["table"], {"cer": 0.10}), | |
| ] | |
| ), | |
| encoding="utf-8", | |
| ) | |
| config = tmp_path / "config.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "ours", | |
| "min_samples": 2, | |
| "bootstrap_iterations": 0, | |
| "gates": [ | |
| { | |
| "baseline": "surya", | |
| "metric": "cer", | |
| "direction": "lower", | |
| "slices": ["hard_eval", "table"], | |
| "min_pairs": 1, | |
| } | |
| ], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| validation = validate_claim(report, tmp_path / "validation", config_path=config) | |
| assert validation.passed | |
| assert validation.gates[0].pairs == 1 | |
| assert validation.gates[0].candidate_mean == 0.01 | |
| def test_validate_claim_supports_sign_test_gate(tmp_path: Path) -> None: | |
| report = tmp_path / "report.json" | |
| report.write_text( | |
| json.dumps( | |
| [ | |
| _row("ours", "s1", "a.png", ["nepali"], {"cer": 0.01}), | |
| _row("tesseract", "s1", "a.png", ["nepali"], {"cer": 0.10}), | |
| _row("ours", "s2", "b.png", ["nepali"], {"cer": 0.01}), | |
| _row("tesseract", "s2", "b.png", ["nepali"], {"cer": 0.10}), | |
| _row("ours", "s3", "c.png", ["nepali"], {"cer": 0.01}), | |
| _row("tesseract", "s3", "c.png", ["nepali"], {"cer": 0.10}), | |
| _row("ours", "s4", "d.png", ["nepali"], {"cer": 0.01}), | |
| _row("tesseract", "s4", "d.png", ["nepali"], {"cer": 0.10}), | |
| _row("ours", "s5", "e.png", ["nepali"], {"cer": 0.01}), | |
| _row("tesseract", "s5", "e.png", ["nepali"], {"cer": 0.10}), | |
| _row("ours", "s6", "f.png", ["nepali"], {"cer": 0.01}), | |
| _row("tesseract", "s6", "f.png", ["nepali"], {"cer": 0.10}), | |
| ], | |
| indent=2, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| config = tmp_path / "config.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "ours", | |
| "min_samples": 6, | |
| "required_slices": {"nepali": 6}, | |
| "bootstrap_iterations": 50, | |
| "gates": [ | |
| { | |
| "baseline": "tesseract", | |
| "metric": "cer", | |
| "direction": "lower", | |
| "slice": "nepali", | |
| "min_pairs": 6, | |
| "max_p_value": 0.05, | |
| } | |
| ], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| validation = validate_claim(report, tmp_path / "validation", config_path=config) | |
| assert validation.passed | |
| assert validation.gates[0].sign_test_p_value is not None | |
| assert validation.gates[0].sign_test_p_value <= 0.05 | |
| def test_validate_claim_reports_aggregate_metrics(tmp_path: Path) -> None: | |
| report = tmp_path / "report.json" | |
| report.write_text( | |
| json.dumps( | |
| [ | |
| _row("ours", "s1", "a.png", ["nepali"], {"cer": 0.02, "quality_score": 0.90}), | |
| _row("ours", "s2", "b.png", ["english"], {"cer": 0.04, "quality_score": 0.80}), | |
| _row("tesseract", "s1", "a.png", ["nepali"], {"cer": 0.10, "quality_score": 0.50}), | |
| _row("tesseract", "s2", "b.png", ["english"], {"cer": 0.20, "quality_score": 0.40}), | |
| ], | |
| indent=2, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| config = tmp_path / "config.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "ours", | |
| "min_samples": 2, | |
| "gates": [{"baseline": "tesseract", "metric": "cer", "direction": "lower", "min_pairs": 2}], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| validation = validate_claim(report, tmp_path / "validation", config_path=config) | |
| assert "ours" in validation.aggregate_metrics | |
| assert validation.aggregate_metrics["ours"]["all"]["sample_count"] == 2 | |
| assert validation.aggregate_metrics["ours"]["nepali"]["cer"] == 0.02 | |
| def test_leakage_check_fails_on_image_hash_and_warns_on_text_only(tmp_path: Path) -> None: | |
| shared_image = tmp_path / "shared.png" | |
| eval_image = tmp_path / "eval.png" | |
| shared_image.write_bytes(b"same") | |
| eval_image.write_bytes(b"different") | |
| image_hash = sha256_file(shared_image) | |
| text_hash = sha256_text("same label") | |
| train_manifest = tmp_path / "train.jsonl" | |
| eval_manifest = tmp_path / "eval.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="train-1", | |
| dataset="local", | |
| split="train", | |
| image_path=str(shared_image), | |
| text="same label", | |
| sha256=image_hash, | |
| metadata={"text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, | |
| ) | |
| ], | |
| train_manifest, | |
| ) | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="eval-1", | |
| dataset="local", | |
| split="eval", | |
| image_path=str(shared_image), | |
| text="same label", | |
| sha256=image_hash, | |
| metadata={"text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, | |
| ), | |
| ManifestEntry( | |
| sample_id="eval-2", | |
| dataset="local", | |
| split="eval", | |
| image_path=str(eval_image), | |
| text="same label", | |
| sha256=sha256_file(eval_image), | |
| metadata={"text_sha256": text_hash}, | |
| ), | |
| ], | |
| eval_manifest, | |
| ) | |
| report = check_leakage([train_manifest], eval_manifest) | |
| assert not report.passed | |
| assert image_hash in report.overlaps["image_sha256"] | |
| assert text_hash in report.warnings["text_sha256"] | |
| def _row(baseline: str, sample_id: str, input_path: str, slices: list[str], metrics: dict[str, float]) -> dict[str, object]: | |
| return { | |
| "baseline": baseline, | |
| "input_path": input_path, | |
| "status": "ok", | |
| "latency_seconds": 1.0, | |
| "sample_id": sample_id, | |
| "slices": slices, | |
| "metrics": metrics, | |
| "error": None, | |
| } | |