File size: 14,423 Bytes
d82db76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
"""Ablation-plan audits for research-grade OCR training branches."""

from __future__ import annotations

import json
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

from .errors import DataValidationError
from .manifest import ManifestEntry, load_manifest


@dataclass(slots=True)
class AugmentationAblationArmAudit:
    name: str
    profiles: list[str]
    train_manifest: str
    eval_manifest: str
    audit_path: str
    train_count: int | None
    eval_count: int | None
    train_manifest_count: int | None = None
    eval_manifest_count: int | None = None
    train_claim_eligible_count: int = 0
    eval_claim_eligible_count: int = 0
    passed: bool = False
    issues: list[str] = field(default_factory=list)
    warnings: list[str] = field(default_factory=list)

    def to_dict(self) -> dict[str, Any]:
        return {
            "name": self.name,
            "profiles": self.profiles,
            "train_manifest": self.train_manifest,
            "eval_manifest": self.eval_manifest,
            "audit_path": self.audit_path,
            "train_count": self.train_count,
            "eval_count": self.eval_count,
            "train_manifest_count": self.train_manifest_count,
            "eval_manifest_count": self.eval_manifest_count,
            "train_claim_eligible_count": self.train_claim_eligible_count,
            "eval_claim_eligible_count": self.eval_claim_eligible_count,
            "passed": self.passed,
            "issues": self.issues,
            "warnings": self.warnings,
        }


@dataclass(slots=True)
class AugmentationAblationAudit:
    config_path: str
    output_json_path: str
    output_md_path: str
    ablation_id: str
    passed: bool
    promotion_ready: bool
    arm_count: int
    real_review_audits: list[str]
    real_evidence_count: int
    verified_reference_count: int
    arms: list[AugmentationAblationArmAudit]
    issues: list[str] = field(default_factory=list)
    warnings: list[str] = field(default_factory=list)

    def to_dict(self) -> dict[str, Any]:
        return {
            "config_path": self.config_path,
            "output_json_path": self.output_json_path,
            "output_md_path": self.output_md_path,
            "ablation_id": self.ablation_id,
            "passed": self.passed,
            "promotion_ready": self.promotion_ready,
            "arm_count": self.arm_count,
            "real_review_audits": self.real_review_audits,
            "real_evidence_count": self.real_evidence_count,
            "verified_reference_count": self.verified_reference_count,
            "arms": [arm.to_dict() for arm in self.arms],
            "issues": self.issues,
            "warnings": self.warnings,
        }


def audit_augmentation_ablation(
    config_path: str | Path,
    output_dir: str | Path,
    *,
    real_review_audits: list[str | Path] | None = None,
) -> AugmentationAblationAudit:
    config_file = Path(config_path)
    config = _read_json_object(config_file, "augmentation ablation config")
    base_dir = config_file.parent
    issues: list[str] = []
    warnings: list[str] = []

    ablation_id = _clean_str(config.get("ablation_id")) or config_file.stem
    claim_policy = config.get("claim_policy")
    if not isinstance(claim_policy, dict):
        issues.append("claim_policy must be an object")
    elif claim_policy.get("claim_evidence_eligible") is not False:
        issues.append("claim_policy.claim_evidence_eligible must be false for synthetic ablation configs")

    source_bundle = config.get("source_bundle")
    if not isinstance(source_bundle, dict):
        issues.append("source_bundle must be an object")
    else:
        _audit_optional_passed_json(
            source_bundle.get("full_split_audit"),
            base_dir,
            "source_bundle.full_split_audit",
            issues,
            warnings,
            require_zero_issues=True,
        )
        font_readiness = source_bundle.get("font_readiness_report")
        if font_readiness:
            _audit_optional_passed_json(font_readiness, base_dir, "source_bundle.font_readiness_report", issues, warnings)

    arms_payload = config.get("arms")
    arms: list[AugmentationAblationArmAudit] = []
    if not isinstance(arms_payload, list) or not arms_payload:
        issues.append("arms must be a non-empty list")
    else:
        for index, payload in enumerate(arms_payload, start=1):
            if not isinstance(payload, dict):
                arm = AugmentationAblationArmAudit(
                    name=f"arm-{index}",
                    profiles=[],
                    train_manifest="",
                    eval_manifest="",
                    audit_path="",
                    train_count=None,
                    eval_count=None,
                    issues=[f"arms[{index}] must be an object"],
                )
                arms.append(arm)
                continue
            arms.append(_audit_ablation_arm(payload, base_dir, index))

    real_paths = [Path(path) for path in (real_review_audits or [])]
    real_issues, real_warnings, real_evidence_count, verified_reference_count = _audit_real_review_evidence(real_paths)
    issues.extend(real_issues)
    warnings.extend(real_warnings)

    for arm in arms:
        issues.extend(f"{arm.name}: {issue}" for issue in arm.issues)
        warnings.extend(f"{arm.name}: {warning}" for warning in arm.warnings)

    promotion_ready = not issues and real_evidence_count > 0 and verified_reference_count > 0 and all(arm.passed for arm in arms)
    out = Path(output_dir)
    out.mkdir(parents=True, exist_ok=True)
    report = AugmentationAblationAudit(
        config_path=str(config_file),
        output_json_path=str(out / "augmentation-ablation-audit.json"),
        output_md_path=str(out / "augmentation-ablation-audit.md"),
        ablation_id=ablation_id,
        passed=promotion_ready,
        promotion_ready=promotion_ready,
        arm_count=len(arms),
        real_review_audits=[str(path) for path in real_paths],
        real_evidence_count=real_evidence_count,
        verified_reference_count=verified_reference_count,
        arms=arms,
        issues=issues,
        warnings=warnings,
    )
    _write_augmentation_ablation_audit(report)
    return report


def _audit_ablation_arm(payload: dict[str, Any], base_dir: Path, index: int) -> AugmentationAblationArmAudit:
    issues: list[str] = []
    warnings: list[str] = []
    name = _clean_str(payload.get("name")) or f"arm-{index}"
    profiles = [str(item) for item in payload.get("profiles") or [] if str(item)]
    if not profiles:
        issues.append("profiles must be a non-empty list")
    train_manifest = _clean_str(payload.get("train_manifest"))
    eval_manifest = _clean_str(payload.get("eval_manifest"))
    audit_path = _clean_str(payload.get("audit"))
    train_count = _optional_int(payload.get("train_count"))
    eval_count = _optional_int(payload.get("eval_count"))

    train_manifest_count, train_claims = _audit_arm_manifest(train_manifest, base_dir, "train_manifest", train_count, issues)
    eval_manifest_count, eval_claims = _audit_arm_manifest(eval_manifest, base_dir, "eval_manifest", eval_count, issues)
    _audit_optional_passed_json(audit_path, base_dir, "audit", issues, warnings, require_zero_issues=True)

    if train_claims:
        issues.append(f"train_manifest has {train_claims} claim_evidence_eligible synthetic row(s)")
    if eval_claims:
        issues.append(f"eval_manifest has {eval_claims} claim_evidence_eligible synthetic row(s)")
    return AugmentationAblationArmAudit(
        name=name,
        profiles=profiles,
        train_manifest=train_manifest,
        eval_manifest=eval_manifest,
        audit_path=audit_path,
        train_count=train_count,
        eval_count=eval_count,
        train_manifest_count=train_manifest_count,
        eval_manifest_count=eval_manifest_count,
        train_claim_eligible_count=train_claims,
        eval_claim_eligible_count=eval_claims,
        passed=not issues,
        issues=issues,
        warnings=warnings,
    )


def _audit_arm_manifest(
    path_value: str,
    base_dir: Path,
    field_name: str,
    expected_count: int | None,
    issues: list[str],
) -> tuple[int | None, int]:
    if not path_value:
        issues.append(f"{field_name} is required")
        return None, 0
    path = _resolve_config_path(path_value, base_dir)
    if not path.is_file():
        issues.append(f"{field_name} does not exist: {path}")
        return None, 0
    try:
        entries = load_manifest(path)
    except Exception as exc:
        issues.append(f"{field_name} failed to load: {path}: {exc}")
        return None, 0
    if expected_count is not None and len(entries) != expected_count:
        issues.append(f"{field_name} count mismatch: expected {expected_count}, found {len(entries)}")
    claim_count = sum(1 for entry in entries if _claim_evidence_eligible(entry))
    return len(entries), claim_count


def _audit_real_review_evidence(paths: list[Path]) -> tuple[list[str], list[str], int, int]:
    issues: list[str] = []
    warnings: list[str] = []
    accepted_total = 0
    verified_total = 0
    if not paths:
        issues.append("at least one real Gorkhapatra review audit is required before ablation promotion")
        return issues, warnings, accepted_total, verified_total
    for path in paths:
        try:
            payload = _read_json_object(path, "Gorkhapatra review audit")
        except DataValidationError as exc:
            issues.append(str(exc))
            continue
        if payload.get("passed") is not True:
            issues.append(f"real review audit did not pass: {path}")
        if payload.get("require_verified_references") is not True:
            issues.append(f"real review audit was not run with --require-verified-references: {path}")
        accepted = _optional_int(payload.get("accepted_count")) or 0
        verified = _optional_int(payload.get("verified_reference_count")) or 0
        if accepted < 1:
            issues.append(f"real review audit has no accepted rows: {path}")
        if verified < accepted:
            issues.append(f"real review audit verified_reference_count is below accepted_count: {path}")
        accepted_total += accepted
        verified_total += verified
        for warning in payload.get("warnings") or []:
            warnings.append(f"{path}: {warning}")
    return issues, warnings, accepted_total, verified_total


def _audit_optional_passed_json(
    path_value: Any,
    base_dir: Path,
    field_name: str,
    issues: list[str],
    warnings: list[str],
    *,
    require_zero_issues: bool = False,
) -> None:
    path_string = _clean_str(path_value)
    if not path_string:
        issues.append(f"{field_name} is required")
        return
    path = _resolve_config_path(path_string, base_dir)
    try:
        payload = _read_json_object(path, field_name)
    except DataValidationError as exc:
        issues.append(str(exc))
        return
    if payload.get("passed") is not True:
        issues.append(f"{field_name} did not pass: {path}")
    if require_zero_issues and payload.get("issues"):
        issues.append(f"{field_name} has issues: {path}")
    for warning in payload.get("warnings") or []:
        warnings.append(f"{field_name}: {path}: {warning}")


def _read_json_object(path: Path, label: str) -> dict[str, Any]:
    if not path.is_file():
        raise DataValidationError(f"{label} does not exist: {path}")
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except json.JSONDecodeError as exc:
        raise DataValidationError(f"{label} is invalid JSON: {path}: {exc}") from exc
    if not isinstance(payload, dict):
        raise DataValidationError(f"{label} must be a JSON object: {path}")
    return payload


def _resolve_config_path(path_value: str, base_dir: Path) -> Path:
    path = Path(path_value)
    if path.is_absolute():
        return path
    if path.exists():
        return path
    return base_dir / path


def _clean_str(value: Any) -> str:
    return value.strip() if isinstance(value, str) else ""


def _optional_int(value: Any) -> int | None:
    if isinstance(value, bool):
        return None
    if isinstance(value, int):
        return value
    return None


def _claim_evidence_eligible(entry: ManifestEntry) -> bool:
    return (entry.metadata or {}).get("claim_evidence_eligible") is not False


def _write_augmentation_ablation_audit(report: AugmentationAblationAudit) -> None:
    json_path = Path(report.output_json_path)
    md_path = Path(report.output_md_path)
    json_path.write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    lines = [
        "# Augmentation Ablation Audit",
        "",
        f"- ablation: `{report.ablation_id}`",
        f"- passed: `{report.passed}`",
        f"- promotion ready: `{report.promotion_ready}`",
        f"- arms: `{report.arm_count}`",
        f"- real review audits: `{len(report.real_review_audits)}`",
        f"- real accepted rows: `{report.real_evidence_count}`",
        f"- verified references: `{report.verified_reference_count}`",
        "",
        "## Arms",
        "",
    ]
    for arm in report.arms:
        lines.extend(
            [
                f"### {arm.name}",
                "",
                f"- passed: `{arm.passed}`",
                f"- profiles: `{', '.join(arm.profiles)}`",
                f"- train rows: `{arm.train_manifest_count if arm.train_manifest_count is not None else 'unknown'}`",
                f"- eval rows: `{arm.eval_manifest_count if arm.eval_manifest_count is not None else 'unknown'}`",
                f"- train claim eligible rows: `{arm.train_claim_eligible_count}`",
                f"- eval claim eligible rows: `{arm.eval_claim_eligible_count}`",
                "",
            ]
        )
        if arm.issues:
            lines.extend(["Issues:", ""])
            lines.extend(f"- {issue}" for issue in arm.issues)
            lines.append("")
    if report.issues:
        lines.extend(["## Issues", ""])
        lines.extend(f"- {issue}" for issue in report.issues)
    if report.warnings:
        lines.extend(["", "## Warnings", ""])
        lines.extend(f"- {warning}" for warning in report.warnings)
    md_path.write_text("\n".join(lines) + "\n", encoding="utf-8")