Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/tests/test_ablation.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 5.1 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/tests/test_ablation.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/tests/test_ablation.py
-
curl -L -o test_ablation.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/tests/test_ablation.py
5.1 kB
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| from ocrtech.ablation import audit_augmentation_ablation | |
| from ocrtech.cli import main | |
| from ocrtech.manifest import ManifestEntry, sha256_text, write_manifest | |
| def test_audit_augmentation_ablation_rejects_synthetic_only_promotion(tmp_path: Path, capsys) -> None: | |
| config = _write_ablation_fixture(tmp_path) | |
| exit_code = main(["audit-augmentation-ablation", str(config), "--out", str(tmp_path / "audit")]) | |
| captured = capsys.readouterr() | |
| payload = json.loads((tmp_path / "audit" / "augmentation-ablation-audit.json").read_text(encoding="utf-8")) | |
| assert exit_code == 1 | |
| assert "passed=False" in captured.out | |
| assert payload["passed"] is False | |
| assert payload["promotion_ready"] is False | |
| assert payload["arm_count"] == 1 | |
| assert payload["arms"][0]["passed"] is True | |
| assert any("real Gorkhapatra review audit is required" in issue for issue in payload["issues"]) | |
| def test_audit_augmentation_ablation_passes_with_verified_real_review_audit(tmp_path: Path) -> None: | |
| config = _write_ablation_fixture(tmp_path) | |
| real_audit = tmp_path / "real-review-audit.json" | |
| real_audit.write_text( | |
| json.dumps( | |
| { | |
| "passed": True, | |
| "require_verified_references": True, | |
| "accepted_count": 1, | |
| "verified_reference_count": 1, | |
| "warnings": [], | |
| }, | |
| ensure_ascii=False, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| report = audit_augmentation_ablation(config, tmp_path / "audit", real_review_audits=[real_audit]) | |
| assert report.passed | |
| assert report.promotion_ready | |
| assert report.real_evidence_count == 1 | |
| assert report.verified_reference_count == 1 | |
| assert report.arms[0].train_manifest_count == 2 | |
| assert report.arms[0].eval_manifest_count == 1 | |
| def test_audit_augmentation_ablation_rejects_claim_eligible_synthetic_arm_rows(tmp_path: Path) -> None: | |
| config = _write_ablation_fixture(tmp_path, claim_eligible_train=True) | |
| real_audit = tmp_path / "real-review-audit.json" | |
| real_audit.write_text( | |
| json.dumps( | |
| { | |
| "passed": True, | |
| "require_verified_references": True, | |
| "accepted_count": 1, | |
| "verified_reference_count": 1, | |
| }, | |
| ensure_ascii=False, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| report = audit_augmentation_ablation(config, tmp_path / "audit", real_review_audits=[real_audit]) | |
| assert not report.passed | |
| assert report.arms[0].train_claim_eligible_count == 1 | |
| assert any("claim_evidence_eligible" in issue for issue in report.issues) | |
| def _write_ablation_fixture(tmp_path: Path, *, claim_eligible_train: bool = False) -> Path: | |
| train_manifest = tmp_path / "train.jsonl" | |
| eval_manifest = tmp_path / "eval.jsonl" | |
| write_manifest( | |
| [ | |
| _manifest_entry("train-1", claim_eligible=claim_eligible_train), | |
| _manifest_entry("train-2", claim_eligible=False), | |
| ], | |
| train_manifest, | |
| ) | |
| write_manifest([_manifest_entry("eval-1", claim_eligible=False)], eval_manifest) | |
| full_audit = tmp_path / "full-split-audit.json" | |
| full_audit.write_text(json.dumps({"passed": True, "issues": [], "warnings": []}) + "\n", encoding="utf-8") | |
| font_readiness = tmp_path / "font-readiness.json" | |
| font_readiness.write_text(json.dumps({"passed": True, "warnings": []}) + "\n", encoding="utf-8") | |
| arm_audit = tmp_path / "arm-audit.json" | |
| arm_audit.write_text(json.dumps({"passed": True, "issues": [], "warnings": []}) + "\n", encoding="utf-8") | |
| config = tmp_path / "ablation.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "ablation_id": "fixture-ablation", | |
| "claim_policy": {"claim_evidence_eligible": False}, | |
| "source_bundle": { | |
| "full_split_audit": str(full_audit), | |
| "font_readiness_report": str(font_readiness), | |
| }, | |
| "arms": [ | |
| { | |
| "name": "clean", | |
| "profiles": ["clean"], | |
| "train_manifest": str(train_manifest), | |
| "eval_manifest": str(eval_manifest), | |
| "audit": str(arm_audit), | |
| "train_count": 2, | |
| "eval_count": 1, | |
| } | |
| ], | |
| }, | |
| ensure_ascii=False, | |
| ) | |
| + "\n", | |
| encoding="utf-8", | |
| ) | |
| return config | |
| def _manifest_entry(sample_id: str, *, claim_eligible: bool) -> ManifestEntry: | |
| text = "ᤕᤠᤰᤌᤢᤱ" | |
| return ManifestEntry( | |
| sample_id=sample_id, | |
| dataset="fixture", | |
| split="train", | |
| image_path=f"{sample_id}.png", | |
| text=text, | |
| sha256="a" * 64, | |
| metadata={ | |
| "source_kind": "rendered_synthesis_text_line", | |
| "claim_evidence_eligible": claim_eligible, | |
| "text_sha256": sha256_text(text), | |
| }, | |
| ) | |