from __future__ import annotations import json from pathlib import Path from ocrtech.ablation import audit_augmentation_ablation from ocrtech.cli import main from ocrtech.manifest import ManifestEntry, sha256_text, write_manifest def test_audit_augmentation_ablation_rejects_synthetic_only_promotion(tmp_path: Path, capsys) -> None: config = _write_ablation_fixture(tmp_path) exit_code = main(["audit-augmentation-ablation", str(config), "--out", str(tmp_path / "audit")]) captured = capsys.readouterr() payload = json.loads((tmp_path / "audit" / "augmentation-ablation-audit.json").read_text(encoding="utf-8")) assert exit_code == 1 assert "passed=False" in captured.out assert payload["passed"] is False assert payload["promotion_ready"] is False assert payload["arm_count"] == 1 assert payload["arms"][0]["passed"] is True assert any("real Gorkhapatra review audit is required" in issue for issue in payload["issues"]) def test_audit_augmentation_ablation_passes_with_verified_real_review_audit(tmp_path: Path) -> None: config = _write_ablation_fixture(tmp_path) real_audit = tmp_path / "real-review-audit.json" real_audit.write_text( json.dumps( { "passed": True, "require_verified_references": True, "accepted_count": 1, "verified_reference_count": 1, "warnings": [], }, ensure_ascii=False, ), encoding="utf-8", ) report = audit_augmentation_ablation(config, tmp_path / "audit", real_review_audits=[real_audit]) assert report.passed assert report.promotion_ready assert report.real_evidence_count == 1 assert report.verified_reference_count == 1 assert report.arms[0].train_manifest_count == 2 assert report.arms[0].eval_manifest_count == 1 def test_audit_augmentation_ablation_rejects_claim_eligible_synthetic_arm_rows(tmp_path: Path) -> None: config = _write_ablation_fixture(tmp_path, claim_eligible_train=True) real_audit = tmp_path / "real-review-audit.json" real_audit.write_text( json.dumps( { "passed": True, "require_verified_references": True, "accepted_count": 1, "verified_reference_count": 1, }, ensure_ascii=False, ), encoding="utf-8", ) report = audit_augmentation_ablation(config, tmp_path / "audit", real_review_audits=[real_audit]) assert not report.passed assert report.arms[0].train_claim_eligible_count == 1 assert any("claim_evidence_eligible" in issue for issue in report.issues) def _write_ablation_fixture(tmp_path: Path, *, claim_eligible_train: bool = False) -> Path: train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" write_manifest( [ _manifest_entry("train-1", claim_eligible=claim_eligible_train), _manifest_entry("train-2", claim_eligible=False), ], train_manifest, ) write_manifest([_manifest_entry("eval-1", claim_eligible=False)], eval_manifest) full_audit = tmp_path / "full-split-audit.json" full_audit.write_text(json.dumps({"passed": True, "issues": [], "warnings": []}) + "\n", encoding="utf-8") font_readiness = tmp_path / "font-readiness.json" font_readiness.write_text(json.dumps({"passed": True, "warnings": []}) + "\n", encoding="utf-8") arm_audit = tmp_path / "arm-audit.json" arm_audit.write_text(json.dumps({"passed": True, "issues": [], "warnings": []}) + "\n", encoding="utf-8") config = tmp_path / "ablation.json" config.write_text( json.dumps( { "ablation_id": "fixture-ablation", "claim_policy": {"claim_evidence_eligible": False}, "source_bundle": { "full_split_audit": str(full_audit), "font_readiness_report": str(font_readiness), }, "arms": [ { "name": "clean", "profiles": ["clean"], "train_manifest": str(train_manifest), "eval_manifest": str(eval_manifest), "audit": str(arm_audit), "train_count": 2, "eval_count": 1, } ], }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) return config def _manifest_entry(sample_id: str, *, claim_eligible: bool) -> ManifestEntry: text = "ᤕᤠᤰᤌᤢᤱ" return ManifestEntry( sample_id=sample_id, dataset="fixture", split="train", image_path=f"{sample_id}.png", text=text, sha256="a" * 64, metadata={ "source_kind": "rendered_synthesis_text_line", "claim_evidence_eligible": claim_eligible, "text_sha256": sha256_text(text), }, )