prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
14.4 kB
"""Ablation-plan audits for research-grade OCR training branches."""
from __future__ import annotations
import json
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from .errors import DataValidationError
from .manifest import ManifestEntry, load_manifest
@dataclass(slots=True)
class AugmentationAblationArmAudit:
name: str
profiles: list[str]
train_manifest: str
eval_manifest: str
audit_path: str
train_count: int | None
eval_count: int | None
train_manifest_count: int | None = None
eval_manifest_count: int | None = None
train_claim_eligible_count: int = 0
eval_claim_eligible_count: int = 0
passed: bool = False
issues: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"name": self.name,
"profiles": self.profiles,
"train_manifest": self.train_manifest,
"eval_manifest": self.eval_manifest,
"audit_path": self.audit_path,
"train_count": self.train_count,
"eval_count": self.eval_count,
"train_manifest_count": self.train_manifest_count,
"eval_manifest_count": self.eval_manifest_count,
"train_claim_eligible_count": self.train_claim_eligible_count,
"eval_claim_eligible_count": self.eval_claim_eligible_count,
"passed": self.passed,
"issues": self.issues,
"warnings": self.warnings,
}
@dataclass(slots=True)
class AugmentationAblationAudit:
config_path: str
output_json_path: str
output_md_path: str
ablation_id: str
passed: bool
promotion_ready: bool
arm_count: int
real_review_audits: list[str]
real_evidence_count: int
verified_reference_count: int
arms: list[AugmentationAblationArmAudit]
issues: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"config_path": self.config_path,
"output_json_path": self.output_json_path,
"output_md_path": self.output_md_path,
"ablation_id": self.ablation_id,
"passed": self.passed,
"promotion_ready": self.promotion_ready,
"arm_count": self.arm_count,
"real_review_audits": self.real_review_audits,
"real_evidence_count": self.real_evidence_count,
"verified_reference_count": self.verified_reference_count,
"arms": [arm.to_dict() for arm in self.arms],
"issues": self.issues,
"warnings": self.warnings,
}
def audit_augmentation_ablation(
config_path: str | Path,
output_dir: str | Path,
*,
real_review_audits: list[str | Path] | None = None,
) -> AugmentationAblationAudit:
config_file = Path(config_path)
config = _read_json_object(config_file, "augmentation ablation config")
base_dir = config_file.parent
issues: list[str] = []
warnings: list[str] = []
ablation_id = _clean_str(config.get("ablation_id")) or config_file.stem
claim_policy = config.get("claim_policy")
if not isinstance(claim_policy, dict):
issues.append("claim_policy must be an object")
elif claim_policy.get("claim_evidence_eligible") is not False:
issues.append("claim_policy.claim_evidence_eligible must be false for synthetic ablation configs")
source_bundle = config.get("source_bundle")
if not isinstance(source_bundle, dict):
issues.append("source_bundle must be an object")
else:
_audit_optional_passed_json(
source_bundle.get("full_split_audit"),
base_dir,
"source_bundle.full_split_audit",
issues,
warnings,
require_zero_issues=True,
)
font_readiness = source_bundle.get("font_readiness_report")
if font_readiness:
_audit_optional_passed_json(font_readiness, base_dir, "source_bundle.font_readiness_report", issues, warnings)
arms_payload = config.get("arms")
arms: list[AugmentationAblationArmAudit] = []
if not isinstance(arms_payload, list) or not arms_payload:
issues.append("arms must be a non-empty list")
else:
for index, payload in enumerate(arms_payload, start=1):
if not isinstance(payload, dict):
arm = AugmentationAblationArmAudit(
name=f"arm-{index}",
profiles=[],
train_manifest="",
eval_manifest="",
audit_path="",
train_count=None,
eval_count=None,
issues=[f"arms[{index}] must be an object"],
)
arms.append(arm)
continue
arms.append(_audit_ablation_arm(payload, base_dir, index))
real_paths = [Path(path) for path in (real_review_audits or [])]
real_issues, real_warnings, real_evidence_count, verified_reference_count = _audit_real_review_evidence(real_paths)
issues.extend(real_issues)
warnings.extend(real_warnings)
for arm in arms:
issues.extend(f"{arm.name}: {issue}" for issue in arm.issues)
warnings.extend(f"{arm.name}: {warning}" for warning in arm.warnings)
promotion_ready = not issues and real_evidence_count > 0 and verified_reference_count > 0 and all(arm.passed for arm in arms)
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
report = AugmentationAblationAudit(
config_path=str(config_file),
output_json_path=str(out / "augmentation-ablation-audit.json"),
output_md_path=str(out / "augmentation-ablation-audit.md"),
ablation_id=ablation_id,
passed=promotion_ready,
promotion_ready=promotion_ready,
arm_count=len(arms),
real_review_audits=[str(path) for path in real_paths],
real_evidence_count=real_evidence_count,
verified_reference_count=verified_reference_count,
arms=arms,
issues=issues,
warnings=warnings,
)
_write_augmentation_ablation_audit(report)
return report
def _audit_ablation_arm(payload: dict[str, Any], base_dir: Path, index: int) -> AugmentationAblationArmAudit:
issues: list[str] = []
warnings: list[str] = []
name = _clean_str(payload.get("name")) or f"arm-{index}"
profiles = [str(item) for item in payload.get("profiles") or [] if str(item)]
if not profiles:
issues.append("profiles must be a non-empty list")
train_manifest = _clean_str(payload.get("train_manifest"))
eval_manifest = _clean_str(payload.get("eval_manifest"))
audit_path = _clean_str(payload.get("audit"))
train_count = _optional_int(payload.get("train_count"))
eval_count = _optional_int(payload.get("eval_count"))
train_manifest_count, train_claims = _audit_arm_manifest(train_manifest, base_dir, "train_manifest", train_count, issues)
eval_manifest_count, eval_claims = _audit_arm_manifest(eval_manifest, base_dir, "eval_manifest", eval_count, issues)
_audit_optional_passed_json(audit_path, base_dir, "audit", issues, warnings, require_zero_issues=True)
if train_claims:
issues.append(f"train_manifest has {train_claims} claim_evidence_eligible synthetic row(s)")
if eval_claims:
issues.append(f"eval_manifest has {eval_claims} claim_evidence_eligible synthetic row(s)")
return AugmentationAblationArmAudit(
name=name,
profiles=profiles,
train_manifest=train_manifest,
eval_manifest=eval_manifest,
audit_path=audit_path,
train_count=train_count,
eval_count=eval_count,
train_manifest_count=train_manifest_count,
eval_manifest_count=eval_manifest_count,
train_claim_eligible_count=train_claims,
eval_claim_eligible_count=eval_claims,
passed=not issues,
issues=issues,
warnings=warnings,
)
def _audit_arm_manifest(
path_value: str,
base_dir: Path,
field_name: str,
expected_count: int | None,
issues: list[str],
) -> tuple[int | None, int]:
if not path_value:
issues.append(f"{field_name} is required")
return None, 0
path = _resolve_config_path(path_value, base_dir)
if not path.is_file():
issues.append(f"{field_name} does not exist: {path}")
return None, 0
try:
entries = load_manifest(path)
except Exception as exc:
issues.append(f"{field_name} failed to load: {path}: {exc}")
return None, 0
if expected_count is not None and len(entries) != expected_count:
issues.append(f"{field_name} count mismatch: expected {expected_count}, found {len(entries)}")
claim_count = sum(1 for entry in entries if _claim_evidence_eligible(entry))
return len(entries), claim_count
def _audit_real_review_evidence(paths: list[Path]) -> tuple[list[str], list[str], int, int]:
issues: list[str] = []
warnings: list[str] = []
accepted_total = 0
verified_total = 0
if not paths:
issues.append("at least one real Gorkhapatra review audit is required before ablation promotion")
return issues, warnings, accepted_total, verified_total
for path in paths:
try:
payload = _read_json_object(path, "Gorkhapatra review audit")
except DataValidationError as exc:
issues.append(str(exc))
continue
if payload.get("passed") is not True:
issues.append(f"real review audit did not pass: {path}")
if payload.get("require_verified_references") is not True:
issues.append(f"real review audit was not run with --require-verified-references: {path}")
accepted = _optional_int(payload.get("accepted_count")) or 0
verified = _optional_int(payload.get("verified_reference_count")) or 0
if accepted < 1:
issues.append(f"real review audit has no accepted rows: {path}")
if verified < accepted:
issues.append(f"real review audit verified_reference_count is below accepted_count: {path}")
accepted_total += accepted
verified_total += verified
for warning in payload.get("warnings") or []:
warnings.append(f"{path}: {warning}")
return issues, warnings, accepted_total, verified_total
def _audit_optional_passed_json(
path_value: Any,
base_dir: Path,
field_name: str,
issues: list[str],
warnings: list[str],
*,
require_zero_issues: bool = False,
) -> None:
path_string = _clean_str(path_value)
if not path_string:
issues.append(f"{field_name} is required")
return
path = _resolve_config_path(path_string, base_dir)
try:
payload = _read_json_object(path, field_name)
except DataValidationError as exc:
issues.append(str(exc))
return
if payload.get("passed") is not True:
issues.append(f"{field_name} did not pass: {path}")
if require_zero_issues and payload.get("issues"):
issues.append(f"{field_name} has issues: {path}")
for warning in payload.get("warnings") or []:
warnings.append(f"{field_name}: {path}: {warning}")
def _read_json_object(path: Path, label: str) -> dict[str, Any]:
if not path.is_file():
raise DataValidationError(f"{label} does not exist: {path}")
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise DataValidationError(f"{label} is invalid JSON: {path}: {exc}") from exc
if not isinstance(payload, dict):
raise DataValidationError(f"{label} must be a JSON object: {path}")
return payload
def _resolve_config_path(path_value: str, base_dir: Path) -> Path:
path = Path(path_value)
if path.is_absolute():
return path
if path.exists():
return path
return base_dir / path
def _clean_str(value: Any) -> str:
return value.strip() if isinstance(value, str) else ""
def _optional_int(value: Any) -> int | None:
if isinstance(value, bool):
return None
if isinstance(value, int):
return value
return None
def _claim_evidence_eligible(entry: ManifestEntry) -> bool:
return (entry.metadata or {}).get("claim_evidence_eligible") is not False
def _write_augmentation_ablation_audit(report: AugmentationAblationAudit) -> None:
json_path = Path(report.output_json_path)
md_path = Path(report.output_md_path)
json_path.write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
lines = [
"# Augmentation Ablation Audit",
"",
f"- ablation: `{report.ablation_id}`",
f"- passed: `{report.passed}`",
f"- promotion ready: `{report.promotion_ready}`",
f"- arms: `{report.arm_count}`",
f"- real review audits: `{len(report.real_review_audits)}`",
f"- real accepted rows: `{report.real_evidence_count}`",
f"- verified references: `{report.verified_reference_count}`",
"",
"## Arms",
"",
]
for arm in report.arms:
lines.extend(
[
f"### {arm.name}",
"",
f"- passed: `{arm.passed}`",
f"- profiles: `{', '.join(arm.profiles)}`",
f"- train rows: `{arm.train_manifest_count if arm.train_manifest_count is not None else 'unknown'}`",
f"- eval rows: `{arm.eval_manifest_count if arm.eval_manifest_count is not None else 'unknown'}`",
f"- train claim eligible rows: `{arm.train_claim_eligible_count}`",
f"- eval claim eligible rows: `{arm.eval_claim_eligible_count}`",
"",
]
)
if arm.issues:
lines.extend(["Issues:", ""])
lines.extend(f"- {issue}" for issue in arm.issues)
lines.append("")
if report.issues:
lines.extend(["## Issues", ""])
lines.extend(f"- {issue}" for issue in report.issues)
if report.warnings:
lines.extend(["", "## Warnings", ""])
lines.extend(f"- {warning}" for warning in report.warnings)
md_path.write_text("\n".join(lines) + "\n", encoding="utf-8")