Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/src/ocrtech/ablation.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 14.4 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/ablation.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/src/ocrtech/ablation.py
-
curl -L -o ablation.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/ablation.py
14.4 kB
| """Ablation-plan audits for research-grade OCR training branches.""" | |
| from __future__ import annotations | |
| import json | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import Any | |
| from .errors import DataValidationError | |
| from .manifest import ManifestEntry, load_manifest | |
| class AugmentationAblationArmAudit: | |
| name: str | |
| profiles: list[str] | |
| train_manifest: str | |
| eval_manifest: str | |
| audit_path: str | |
| train_count: int | None | |
| eval_count: int | None | |
| train_manifest_count: int | None = None | |
| eval_manifest_count: int | None = None | |
| train_claim_eligible_count: int = 0 | |
| eval_claim_eligible_count: int = 0 | |
| passed: bool = False | |
| issues: list[str] = field(default_factory=list) | |
| warnings: list[str] = field(default_factory=list) | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "name": self.name, | |
| "profiles": self.profiles, | |
| "train_manifest": self.train_manifest, | |
| "eval_manifest": self.eval_manifest, | |
| "audit_path": self.audit_path, | |
| "train_count": self.train_count, | |
| "eval_count": self.eval_count, | |
| "train_manifest_count": self.train_manifest_count, | |
| "eval_manifest_count": self.eval_manifest_count, | |
| "train_claim_eligible_count": self.train_claim_eligible_count, | |
| "eval_claim_eligible_count": self.eval_claim_eligible_count, | |
| "passed": self.passed, | |
| "issues": self.issues, | |
| "warnings": self.warnings, | |
| } | |
| class AugmentationAblationAudit: | |
| config_path: str | |
| output_json_path: str | |
| output_md_path: str | |
| ablation_id: str | |
| passed: bool | |
| promotion_ready: bool | |
| arm_count: int | |
| real_review_audits: list[str] | |
| real_evidence_count: int | |
| verified_reference_count: int | |
| arms: list[AugmentationAblationArmAudit] | |
| issues: list[str] = field(default_factory=list) | |
| warnings: list[str] = field(default_factory=list) | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "config_path": self.config_path, | |
| "output_json_path": self.output_json_path, | |
| "output_md_path": self.output_md_path, | |
| "ablation_id": self.ablation_id, | |
| "passed": self.passed, | |
| "promotion_ready": self.promotion_ready, | |
| "arm_count": self.arm_count, | |
| "real_review_audits": self.real_review_audits, | |
| "real_evidence_count": self.real_evidence_count, | |
| "verified_reference_count": self.verified_reference_count, | |
| "arms": [arm.to_dict() for arm in self.arms], | |
| "issues": self.issues, | |
| "warnings": self.warnings, | |
| } | |
| def audit_augmentation_ablation( | |
| config_path: str | Path, | |
| output_dir: str | Path, | |
| *, | |
| real_review_audits: list[str | Path] | None = None, | |
| ) -> AugmentationAblationAudit: | |
| config_file = Path(config_path) | |
| config = _read_json_object(config_file, "augmentation ablation config") | |
| base_dir = config_file.parent | |
| issues: list[str] = [] | |
| warnings: list[str] = [] | |
| ablation_id = _clean_str(config.get("ablation_id")) or config_file.stem | |
| claim_policy = config.get("claim_policy") | |
| if not isinstance(claim_policy, dict): | |
| issues.append("claim_policy must be an object") | |
| elif claim_policy.get("claim_evidence_eligible") is not False: | |
| issues.append("claim_policy.claim_evidence_eligible must be false for synthetic ablation configs") | |
| source_bundle = config.get("source_bundle") | |
| if not isinstance(source_bundle, dict): | |
| issues.append("source_bundle must be an object") | |
| else: | |
| _audit_optional_passed_json( | |
| source_bundle.get("full_split_audit"), | |
| base_dir, | |
| "source_bundle.full_split_audit", | |
| issues, | |
| warnings, | |
| require_zero_issues=True, | |
| ) | |
| font_readiness = source_bundle.get("font_readiness_report") | |
| if font_readiness: | |
| _audit_optional_passed_json(font_readiness, base_dir, "source_bundle.font_readiness_report", issues, warnings) | |
| arms_payload = config.get("arms") | |
| arms: list[AugmentationAblationArmAudit] = [] | |
| if not isinstance(arms_payload, list) or not arms_payload: | |
| issues.append("arms must be a non-empty list") | |
| else: | |
| for index, payload in enumerate(arms_payload, start=1): | |
| if not isinstance(payload, dict): | |
| arm = AugmentationAblationArmAudit( | |
| name=f"arm-{index}", | |
| profiles=[], | |
| train_manifest="", | |
| eval_manifest="", | |
| audit_path="", | |
| train_count=None, | |
| eval_count=None, | |
| issues=[f"arms[{index}] must be an object"], | |
| ) | |
| arms.append(arm) | |
| continue | |
| arms.append(_audit_ablation_arm(payload, base_dir, index)) | |
| real_paths = [Path(path) for path in (real_review_audits or [])] | |
| real_issues, real_warnings, real_evidence_count, verified_reference_count = _audit_real_review_evidence(real_paths) | |
| issues.extend(real_issues) | |
| warnings.extend(real_warnings) | |
| for arm in arms: | |
| issues.extend(f"{arm.name}: {issue}" for issue in arm.issues) | |
| warnings.extend(f"{arm.name}: {warning}" for warning in arm.warnings) | |
| promotion_ready = not issues and real_evidence_count > 0 and verified_reference_count > 0 and all(arm.passed for arm in arms) | |
| out = Path(output_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| report = AugmentationAblationAudit( | |
| config_path=str(config_file), | |
| output_json_path=str(out / "augmentation-ablation-audit.json"), | |
| output_md_path=str(out / "augmentation-ablation-audit.md"), | |
| ablation_id=ablation_id, | |
| passed=promotion_ready, | |
| promotion_ready=promotion_ready, | |
| arm_count=len(arms), | |
| real_review_audits=[str(path) for path in real_paths], | |
| real_evidence_count=real_evidence_count, | |
| verified_reference_count=verified_reference_count, | |
| arms=arms, | |
| issues=issues, | |
| warnings=warnings, | |
| ) | |
| _write_augmentation_ablation_audit(report) | |
| return report | |
| def _audit_ablation_arm(payload: dict[str, Any], base_dir: Path, index: int) -> AugmentationAblationArmAudit: | |
| issues: list[str] = [] | |
| warnings: list[str] = [] | |
| name = _clean_str(payload.get("name")) or f"arm-{index}" | |
| profiles = [str(item) for item in payload.get("profiles") or [] if str(item)] | |
| if not profiles: | |
| issues.append("profiles must be a non-empty list") | |
| train_manifest = _clean_str(payload.get("train_manifest")) | |
| eval_manifest = _clean_str(payload.get("eval_manifest")) | |
| audit_path = _clean_str(payload.get("audit")) | |
| train_count = _optional_int(payload.get("train_count")) | |
| eval_count = _optional_int(payload.get("eval_count")) | |
| train_manifest_count, train_claims = _audit_arm_manifest(train_manifest, base_dir, "train_manifest", train_count, issues) | |
| eval_manifest_count, eval_claims = _audit_arm_manifest(eval_manifest, base_dir, "eval_manifest", eval_count, issues) | |
| _audit_optional_passed_json(audit_path, base_dir, "audit", issues, warnings, require_zero_issues=True) | |
| if train_claims: | |
| issues.append(f"train_manifest has {train_claims} claim_evidence_eligible synthetic row(s)") | |
| if eval_claims: | |
| issues.append(f"eval_manifest has {eval_claims} claim_evidence_eligible synthetic row(s)") | |
| return AugmentationAblationArmAudit( | |
| name=name, | |
| profiles=profiles, | |
| train_manifest=train_manifest, | |
| eval_manifest=eval_manifest, | |
| audit_path=audit_path, | |
| train_count=train_count, | |
| eval_count=eval_count, | |
| train_manifest_count=train_manifest_count, | |
| eval_manifest_count=eval_manifest_count, | |
| train_claim_eligible_count=train_claims, | |
| eval_claim_eligible_count=eval_claims, | |
| passed=not issues, | |
| issues=issues, | |
| warnings=warnings, | |
| ) | |
| def _audit_arm_manifest( | |
| path_value: str, | |
| base_dir: Path, | |
| field_name: str, | |
| expected_count: int | None, | |
| issues: list[str], | |
| ) -> tuple[int | None, int]: | |
| if not path_value: | |
| issues.append(f"{field_name} is required") | |
| return None, 0 | |
| path = _resolve_config_path(path_value, base_dir) | |
| if not path.is_file(): | |
| issues.append(f"{field_name} does not exist: {path}") | |
| return None, 0 | |
| try: | |
| entries = load_manifest(path) | |
| except Exception as exc: | |
| issues.append(f"{field_name} failed to load: {path}: {exc}") | |
| return None, 0 | |
| if expected_count is not None and len(entries) != expected_count: | |
| issues.append(f"{field_name} count mismatch: expected {expected_count}, found {len(entries)}") | |
| claim_count = sum(1 for entry in entries if _claim_evidence_eligible(entry)) | |
| return len(entries), claim_count | |
| def _audit_real_review_evidence(paths: list[Path]) -> tuple[list[str], list[str], int, int]: | |
| issues: list[str] = [] | |
| warnings: list[str] = [] | |
| accepted_total = 0 | |
| verified_total = 0 | |
| if not paths: | |
| issues.append("at least one real Gorkhapatra review audit is required before ablation promotion") | |
| return issues, warnings, accepted_total, verified_total | |
| for path in paths: | |
| try: | |
| payload = _read_json_object(path, "Gorkhapatra review audit") | |
| except DataValidationError as exc: | |
| issues.append(str(exc)) | |
| continue | |
| if payload.get("passed") is not True: | |
| issues.append(f"real review audit did not pass: {path}") | |
| if payload.get("require_verified_references") is not True: | |
| issues.append(f"real review audit was not run with --require-verified-references: {path}") | |
| accepted = _optional_int(payload.get("accepted_count")) or 0 | |
| verified = _optional_int(payload.get("verified_reference_count")) or 0 | |
| if accepted < 1: | |
| issues.append(f"real review audit has no accepted rows: {path}") | |
| if verified < accepted: | |
| issues.append(f"real review audit verified_reference_count is below accepted_count: {path}") | |
| accepted_total += accepted | |
| verified_total += verified | |
| for warning in payload.get("warnings") or []: | |
| warnings.append(f"{path}: {warning}") | |
| return issues, warnings, accepted_total, verified_total | |
| def _audit_optional_passed_json( | |
| path_value: Any, | |
| base_dir: Path, | |
| field_name: str, | |
| issues: list[str], | |
| warnings: list[str], | |
| *, | |
| require_zero_issues: bool = False, | |
| ) -> None: | |
| path_string = _clean_str(path_value) | |
| if not path_string: | |
| issues.append(f"{field_name} is required") | |
| return | |
| path = _resolve_config_path(path_string, base_dir) | |
| try: | |
| payload = _read_json_object(path, field_name) | |
| except DataValidationError as exc: | |
| issues.append(str(exc)) | |
| return | |
| if payload.get("passed") is not True: | |
| issues.append(f"{field_name} did not pass: {path}") | |
| if require_zero_issues and payload.get("issues"): | |
| issues.append(f"{field_name} has issues: {path}") | |
| for warning in payload.get("warnings") or []: | |
| warnings.append(f"{field_name}: {path}: {warning}") | |
| def _read_json_object(path: Path, label: str) -> dict[str, Any]: | |
| if not path.is_file(): | |
| raise DataValidationError(f"{label} does not exist: {path}") | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except json.JSONDecodeError as exc: | |
| raise DataValidationError(f"{label} is invalid JSON: {path}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise DataValidationError(f"{label} must be a JSON object: {path}") | |
| return payload | |
| def _resolve_config_path(path_value: str, base_dir: Path) -> Path: | |
| path = Path(path_value) | |
| if path.is_absolute(): | |
| return path | |
| if path.exists(): | |
| return path | |
| return base_dir / path | |
| def _clean_str(value: Any) -> str: | |
| return value.strip() if isinstance(value, str) else "" | |
| def _optional_int(value: Any) -> int | None: | |
| if isinstance(value, bool): | |
| return None | |
| if isinstance(value, int): | |
| return value | |
| return None | |
| def _claim_evidence_eligible(entry: ManifestEntry) -> bool: | |
| return (entry.metadata or {}).get("claim_evidence_eligible") is not False | |
| def _write_augmentation_ablation_audit(report: AugmentationAblationAudit) -> None: | |
| json_path = Path(report.output_json_path) | |
| md_path = Path(report.output_md_path) | |
| json_path.write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| lines = [ | |
| "# Augmentation Ablation Audit", | |
| "", | |
| f"- ablation: `{report.ablation_id}`", | |
| f"- passed: `{report.passed}`", | |
| f"- promotion ready: `{report.promotion_ready}`", | |
| f"- arms: `{report.arm_count}`", | |
| f"- real review audits: `{len(report.real_review_audits)}`", | |
| f"- real accepted rows: `{report.real_evidence_count}`", | |
| f"- verified references: `{report.verified_reference_count}`", | |
| "", | |
| "## Arms", | |
| "", | |
| ] | |
| for arm in report.arms: | |
| lines.extend( | |
| [ | |
| f"### {arm.name}", | |
| "", | |
| f"- passed: `{arm.passed}`", | |
| f"- profiles: `{', '.join(arm.profiles)}`", | |
| f"- train rows: `{arm.train_manifest_count if arm.train_manifest_count is not None else 'unknown'}`", | |
| f"- eval rows: `{arm.eval_manifest_count if arm.eval_manifest_count is not None else 'unknown'}`", | |
| f"- train claim eligible rows: `{arm.train_claim_eligible_count}`", | |
| f"- eval claim eligible rows: `{arm.eval_claim_eligible_count}`", | |
| "", | |
| ] | |
| ) | |
| if arm.issues: | |
| lines.extend(["Issues:", ""]) | |
| lines.extend(f"- {issue}" for issue in arm.issues) | |
| lines.append("") | |
| if report.issues: | |
| lines.extend(["## Issues", ""]) | |
| lines.extend(f"- {issue}" for issue in report.issues) | |
| if report.warnings: | |
| lines.extend(["", "## Warnings", ""]) | |
| lines.extend(f"- {warning}" for warning in report.warnings) | |
| md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") | |