prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
5.1 kB
from __future__ import annotations
import json
from pathlib import Path
from ocrtech.ablation import audit_augmentation_ablation
from ocrtech.cli import main
from ocrtech.manifest import ManifestEntry, sha256_text, write_manifest
def test_audit_augmentation_ablation_rejects_synthetic_only_promotion(tmp_path: Path, capsys) -> None:
config = _write_ablation_fixture(tmp_path)
exit_code = main(["audit-augmentation-ablation", str(config), "--out", str(tmp_path / "audit")])
captured = capsys.readouterr()
payload = json.loads((tmp_path / "audit" / "augmentation-ablation-audit.json").read_text(encoding="utf-8"))
assert exit_code == 1
assert "passed=False" in captured.out
assert payload["passed"] is False
assert payload["promotion_ready"] is False
assert payload["arm_count"] == 1
assert payload["arms"][0]["passed"] is True
assert any("real Gorkhapatra review audit is required" in issue for issue in payload["issues"])
def test_audit_augmentation_ablation_passes_with_verified_real_review_audit(tmp_path: Path) -> None:
config = _write_ablation_fixture(tmp_path)
real_audit = tmp_path / "real-review-audit.json"
real_audit.write_text(
json.dumps(
{
"passed": True,
"require_verified_references": True,
"accepted_count": 1,
"verified_reference_count": 1,
"warnings": [],
},
ensure_ascii=False,
),
encoding="utf-8",
)
report = audit_augmentation_ablation(config, tmp_path / "audit", real_review_audits=[real_audit])
assert report.passed
assert report.promotion_ready
assert report.real_evidence_count == 1
assert report.verified_reference_count == 1
assert report.arms[0].train_manifest_count == 2
assert report.arms[0].eval_manifest_count == 1
def test_audit_augmentation_ablation_rejects_claim_eligible_synthetic_arm_rows(tmp_path: Path) -> None:
config = _write_ablation_fixture(tmp_path, claim_eligible_train=True)
real_audit = tmp_path / "real-review-audit.json"
real_audit.write_text(
json.dumps(
{
"passed": True,
"require_verified_references": True,
"accepted_count": 1,
"verified_reference_count": 1,
},
ensure_ascii=False,
),
encoding="utf-8",
)
report = audit_augmentation_ablation(config, tmp_path / "audit", real_review_audits=[real_audit])
assert not report.passed
assert report.arms[0].train_claim_eligible_count == 1
assert any("claim_evidence_eligible" in issue for issue in report.issues)
def _write_ablation_fixture(tmp_path: Path, *, claim_eligible_train: bool = False) -> Path:
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[
_manifest_entry("train-1", claim_eligible=claim_eligible_train),
_manifest_entry("train-2", claim_eligible=False),
],
train_manifest,
)
write_manifest([_manifest_entry("eval-1", claim_eligible=False)], eval_manifest)
full_audit = tmp_path / "full-split-audit.json"
full_audit.write_text(json.dumps({"passed": True, "issues": [], "warnings": []}) + "\n", encoding="utf-8")
font_readiness = tmp_path / "font-readiness.json"
font_readiness.write_text(json.dumps({"passed": True, "warnings": []}) + "\n", encoding="utf-8")
arm_audit = tmp_path / "arm-audit.json"
arm_audit.write_text(json.dumps({"passed": True, "issues": [], "warnings": []}) + "\n", encoding="utf-8")
config = tmp_path / "ablation.json"
config.write_text(
json.dumps(
{
"ablation_id": "fixture-ablation",
"claim_policy": {"claim_evidence_eligible": False},
"source_bundle": {
"full_split_audit": str(full_audit),
"font_readiness_report": str(font_readiness),
},
"arms": [
{
"name": "clean",
"profiles": ["clean"],
"train_manifest": str(train_manifest),
"eval_manifest": str(eval_manifest),
"audit": str(arm_audit),
"train_count": 2,
"eval_count": 1,
}
],
},
ensure_ascii=False,
)
+ "\n",
encoding="utf-8",
)
return config
def _manifest_entry(sample_id: str, *, claim_eligible: bool) -> ManifestEntry:
text = "ᤕᤠᤰᤌᤢᤱ"
return ManifestEntry(
sample_id=sample_id,
dataset="fixture",
split="train",
image_path=f"{sample_id}.png",
text=text,
sha256="a" * 64,
metadata={
"source_kind": "rendered_synthesis_text_line",
"claim_evidence_eligible": claim_eligible,
"text_sha256": sha256_text(text),
},
)