File size: 5,103 Bytes
f9bc0c1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | from __future__ import annotations
import json
from pathlib import Path
from ocrtech.ablation import audit_augmentation_ablation
from ocrtech.cli import main
from ocrtech.manifest import ManifestEntry, sha256_text, write_manifest
def test_audit_augmentation_ablation_rejects_synthetic_only_promotion(tmp_path: Path, capsys) -> None:
config = _write_ablation_fixture(tmp_path)
exit_code = main(["audit-augmentation-ablation", str(config), "--out", str(tmp_path / "audit")])
captured = capsys.readouterr()
payload = json.loads((tmp_path / "audit" / "augmentation-ablation-audit.json").read_text(encoding="utf-8"))
assert exit_code == 1
assert "passed=False" in captured.out
assert payload["passed"] is False
assert payload["promotion_ready"] is False
assert payload["arm_count"] == 1
assert payload["arms"][0]["passed"] is True
assert any("real Gorkhapatra review audit is required" in issue for issue in payload["issues"])
def test_audit_augmentation_ablation_passes_with_verified_real_review_audit(tmp_path: Path) -> None:
config = _write_ablation_fixture(tmp_path)
real_audit = tmp_path / "real-review-audit.json"
real_audit.write_text(
json.dumps(
{
"passed": True,
"require_verified_references": True,
"accepted_count": 1,
"verified_reference_count": 1,
"warnings": [],
},
ensure_ascii=False,
),
encoding="utf-8",
)
report = audit_augmentation_ablation(config, tmp_path / "audit", real_review_audits=[real_audit])
assert report.passed
assert report.promotion_ready
assert report.real_evidence_count == 1
assert report.verified_reference_count == 1
assert report.arms[0].train_manifest_count == 2
assert report.arms[0].eval_manifest_count == 1
def test_audit_augmentation_ablation_rejects_claim_eligible_synthetic_arm_rows(tmp_path: Path) -> None:
config = _write_ablation_fixture(tmp_path, claim_eligible_train=True)
real_audit = tmp_path / "real-review-audit.json"
real_audit.write_text(
json.dumps(
{
"passed": True,
"require_verified_references": True,
"accepted_count": 1,
"verified_reference_count": 1,
},
ensure_ascii=False,
),
encoding="utf-8",
)
report = audit_augmentation_ablation(config, tmp_path / "audit", real_review_audits=[real_audit])
assert not report.passed
assert report.arms[0].train_claim_eligible_count == 1
assert any("claim_evidence_eligible" in issue for issue in report.issues)
def _write_ablation_fixture(tmp_path: Path, *, claim_eligible_train: bool = False) -> Path:
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[
_manifest_entry("train-1", claim_eligible=claim_eligible_train),
_manifest_entry("train-2", claim_eligible=False),
],
train_manifest,
)
write_manifest([_manifest_entry("eval-1", claim_eligible=False)], eval_manifest)
full_audit = tmp_path / "full-split-audit.json"
full_audit.write_text(json.dumps({"passed": True, "issues": [], "warnings": []}) + "\n", encoding="utf-8")
font_readiness = tmp_path / "font-readiness.json"
font_readiness.write_text(json.dumps({"passed": True, "warnings": []}) + "\n", encoding="utf-8")
arm_audit = tmp_path / "arm-audit.json"
arm_audit.write_text(json.dumps({"passed": True, "issues": [], "warnings": []}) + "\n", encoding="utf-8")
config = tmp_path / "ablation.json"
config.write_text(
json.dumps(
{
"ablation_id": "fixture-ablation",
"claim_policy": {"claim_evidence_eligible": False},
"source_bundle": {
"full_split_audit": str(full_audit),
"font_readiness_report": str(font_readiness),
},
"arms": [
{
"name": "clean",
"profiles": ["clean"],
"train_manifest": str(train_manifest),
"eval_manifest": str(eval_manifest),
"audit": str(arm_audit),
"train_count": 2,
"eval_count": 1,
}
],
},
ensure_ascii=False,
)
+ "\n",
encoding="utf-8",
)
return config
def _manifest_entry(sample_id: str, *, claim_eligible: bool) -> ManifestEntry:
text = "ᤕᤠᤰᤌᤢᤱ"
return ManifestEntry(
sample_id=sample_id,
dataset="fixture",
split="train",
image_path=f"{sample_id}.png",
text=text,
sha256="a" * 64,
metadata={
"source_kind": "rendered_synthesis_text_line",
"claim_evidence_eligible": claim_eligible,
"text_sha256": sha256_text(text),
},
)
|