Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/src/ocrtech/admission.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 29.9 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/admission.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/src/ocrtech/admission.py
-
curl -L -o admission.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/admission.py
29.9 kB
| """Recognizer admission decision artifacts.""" | |
| from __future__ import annotations | |
| import json | |
| import math | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import Any | |
| from .errors import ValidationError | |
| class FailedAdmissionGate: | |
| baseline: str | |
| metric: str | |
| slices: list[str] = field(default_factory=list) | |
| pairs: int | None = None | |
| candidate_mean: float | None = None | |
| baseline_mean: float | None = None | |
| absolute_improvement: float | None = None | |
| relative_improvement: float | None = None | |
| win_count: int | None = None | |
| loss_count: int | None = None | |
| tie_count: int | None = None | |
| reason: str | None = None | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "baseline": self.baseline, | |
| "metric": self.metric, | |
| "slices": self.slices, | |
| "pairs": self.pairs, | |
| "candidate_mean": self.candidate_mean, | |
| "baseline_mean": self.baseline_mean, | |
| "absolute_improvement": self.absolute_improvement, | |
| "relative_improvement": self.relative_improvement, | |
| "win_count": self.win_count, | |
| "loss_count": self.loss_count, | |
| "tie_count": self.tie_count, | |
| "reason": self.reason, | |
| } | |
| class RecognizerAdmissionDecision: | |
| validation_report: str | |
| decision: str | |
| claim_status: str | |
| passed: bool | |
| sample_count: int | None | |
| missing_requirements: list[str] = field(default_factory=list) | |
| failed_gates: list[FailedAdmissionGate] = field(default_factory=list) | |
| next_actions: list[str] = field(default_factory=list) | |
| decision_path: str | None = None | |
| markdown_path: str | None = None | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "validation_report": self.validation_report, | |
| "decision": self.decision, | |
| "claim_status": self.claim_status, | |
| "passed": self.passed, | |
| "sample_count": self.sample_count, | |
| "missing_requirements": self.missing_requirements, | |
| "failed_gates": [gate.to_dict() for gate in self.failed_gates], | |
| "next_actions": self.next_actions, | |
| "decision_path": self.decision_path, | |
| "markdown_path": self.markdown_path, | |
| } | |
| class RecognizerFailureRun: | |
| source_path: str | |
| validation_report: str | None | |
| model_card: str | None | |
| model_id: str | |
| backend: str | |
| base_model: str | |
| decision: str | |
| claim_status: str | |
| passed: bool | |
| sample_count: int | None | |
| failed_gate_count: int | |
| total_pairs: int | |
| total_wins: int | |
| total_losses: int | |
| total_ties: int | |
| failed_slices: list[str] | |
| failed_metrics: list[str] | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "source_path": self.source_path, | |
| "validation_report": self.validation_report, | |
| "model_card": self.model_card, | |
| "model_id": self.model_id, | |
| "backend": self.backend, | |
| "base_model": self.base_model, | |
| "decision": self.decision, | |
| "claim_status": self.claim_status, | |
| "passed": self.passed, | |
| "sample_count": self.sample_count, | |
| "failed_gate_count": self.failed_gate_count, | |
| "total_pairs": self.total_pairs, | |
| "total_wins": self.total_wins, | |
| "total_losses": self.total_losses, | |
| "total_ties": self.total_ties, | |
| "failed_slices": self.failed_slices, | |
| "failed_metrics": self.failed_metrics, | |
| } | |
| class RecognizerFailureFamily: | |
| family_key: str | |
| backend: str | |
| base_model: str | |
| run_count: int | |
| rejected_count: int | |
| admitted_count: int | |
| total_failed_gates: int | |
| total_pairs: int | |
| total_wins: int | |
| total_losses: int | |
| total_ties: int | |
| failed_slices: list[str] | |
| failed_metrics: list[str] | |
| recommendation: str | |
| rationale: list[str] = field(default_factory=list) | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "family_key": self.family_key, | |
| "backend": self.backend, | |
| "base_model": self.base_model, | |
| "run_count": self.run_count, | |
| "rejected_count": self.rejected_count, | |
| "admitted_count": self.admitted_count, | |
| "total_failed_gates": self.total_failed_gates, | |
| "total_pairs": self.total_pairs, | |
| "total_wins": self.total_wins, | |
| "total_losses": self.total_losses, | |
| "total_ties": self.total_ties, | |
| "failed_slices": self.failed_slices, | |
| "failed_metrics": self.failed_metrics, | |
| "recommendation": self.recommendation, | |
| "rationale": self.rationale, | |
| } | |
| class RecognizerFailureReview: | |
| admission_decisions: list[str] | |
| validation_reports: list[str] | |
| model_cards: list[str] | |
| min_rejections_to_stop: int | |
| runs: list[RecognizerFailureRun] | |
| families: list[RecognizerFailureFamily] | |
| next_actions: list[str] | |
| review_path: str | None = None | |
| markdown_path: str | None = None | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "admission_decisions": self.admission_decisions, | |
| "validation_reports": self.validation_reports, | |
| "model_cards": self.model_cards, | |
| "min_rejections_to_stop": self.min_rejections_to_stop, | |
| "runs": [run.to_dict() for run in self.runs], | |
| "families": [family.to_dict() for family in self.families], | |
| "next_actions": self.next_actions, | |
| "review_path": self.review_path, | |
| "markdown_path": self.markdown_path, | |
| } | |
| def decide_recognizer_admission(validation_report: str | Path, output_dir: str | Path) -> RecognizerAdmissionDecision: | |
| report_path = Path(validation_report) | |
| try: | |
| payload = json.loads(report_path.read_text(encoding="utf-8")) | |
| except FileNotFoundError as exc: | |
| raise ValidationError(f"admission validation report does not exist: {report_path}") from exc | |
| except json.JSONDecodeError as exc: | |
| raise ValidationError(f"invalid admission validation JSON {report_path}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise ValidationError(f"admission validation report must be a JSON object: {report_path}") | |
| claim_status = str(payload.get("claim_status") or "") | |
| passed = bool(payload.get("passed")) | |
| decision = "admit" if claim_status == "validated" and passed else "reject" | |
| failed_gates = _failed_gates(payload) | |
| missing_requirements = _missing_requirements(payload) | |
| next_actions = _next_actions(decision, failed_gates, missing_requirements) | |
| sample_count_value = payload.get("sample_count") | |
| sample_count = int(sample_count_value) if isinstance(sample_count_value, int | float) and not isinstance(sample_count_value, bool) else None | |
| out = Path(output_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| decision_path = out / "recognizer-admission-decision.json" | |
| markdown_path = out / "recognizer-admission-decision.md" | |
| decision_report = RecognizerAdmissionDecision( | |
| validation_report=str(report_path), | |
| decision=decision, | |
| claim_status=claim_status or "not_run", | |
| passed=passed, | |
| sample_count=sample_count, | |
| missing_requirements=missing_requirements, | |
| failed_gates=failed_gates, | |
| next_actions=next_actions, | |
| decision_path=str(decision_path), | |
| markdown_path=str(markdown_path), | |
| ) | |
| decision_path.write_text(json.dumps(decision_report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| _write_markdown(markdown_path, decision_report) | |
| return decision_report | |
| def review_recognizer_failures( | |
| admission_decisions: list[str | Path], | |
| validation_reports: list[str | Path], | |
| output_dir: str | Path, | |
| *, | |
| model_cards: list[str | Path] | None = None, | |
| min_rejections_to_stop: int = 3, | |
| ) -> RecognizerFailureReview: | |
| if min_rejections_to_stop < 1: | |
| raise ValidationError("min_rejections_to_stop must be >= 1") | |
| if not admission_decisions and not validation_reports: | |
| raise ValidationError("recognizer failure review requires at least one admission decision or validation report") | |
| cards = [Path(path) for path in (model_cards or [])] | |
| card_payloads = [_read_json_object(path, "model card") for path in cards] | |
| runs: list[RecognizerFailureRun] = [] | |
| for index, path_value in enumerate(admission_decisions): | |
| path = Path(path_value) | |
| decision_payload = _read_json_object(path, "recognizer admission decision") | |
| validation_payload = _read_optional_validation_payload(decision_payload, path) | |
| model_identity = _model_identity_for_run(index, card_payloads, cards, validation_payload, path) | |
| runs.append(_failure_run_from_decision(path, decision_payload, model_identity)) | |
| offset = len(runs) | |
| for index, path_value in enumerate(validation_reports): | |
| path = Path(path_value) | |
| validation_payload = _read_json_object(path, "admission validation report") | |
| decision_payload = _decision_payload_from_validation(path, validation_payload) | |
| model_identity = _model_identity_for_run(offset + index, card_payloads, cards, validation_payload, path) | |
| runs.append(_failure_run_from_decision(path, decision_payload, model_identity)) | |
| families = _failure_families(runs, min_rejections_to_stop=min_rejections_to_stop) | |
| next_actions = _failure_review_next_actions(families) | |
| out = Path(output_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| review_path = out / "recognizer-failure-review.json" | |
| markdown_path = out / "recognizer-failure-review.md" | |
| review = RecognizerFailureReview( | |
| admission_decisions=[str(Path(path)) for path in admission_decisions], | |
| validation_reports=[str(Path(path)) for path in validation_reports], | |
| model_cards=[str(path) for path in cards], | |
| min_rejections_to_stop=min_rejections_to_stop, | |
| runs=runs, | |
| families=families, | |
| next_actions=next_actions, | |
| review_path=str(review_path), | |
| markdown_path=str(markdown_path), | |
| ) | |
| review_path.write_text(json.dumps(review.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| _write_failure_review_markdown(markdown_path, review) | |
| return review | |
| def stopped_recognizer_family_reasons(failure_review: str | Path, *, backend: str, base_model: str) -> list[str]: | |
| review_path = Path(failure_review) | |
| payload = _read_json_object(review_path, "recognizer failure review") | |
| families = payload.get("families") | |
| if not isinstance(families, list): | |
| raise ValidationError(f"recognizer failure review missing families list: {review_path}") | |
| canonical_base = _canonical_recognizer_base_model(base_model) | |
| reasons: list[str] = [] | |
| for item in families: | |
| if not isinstance(item, dict): | |
| continue | |
| family_backend = str(item.get("backend") or "") | |
| family_base = _canonical_recognizer_base_model(str(item.get("base_model") or "")) | |
| if family_backend != backend or family_base != canonical_base: | |
| continue | |
| if item.get("recommendation") != "stop_family": | |
| continue | |
| reasons.append( | |
| "recognizer family is stopped by {review}: family={family} rejected_runs={rejected} " | |
| "wins/losses/ties={wins}/{losses}/{ties}".format( | |
| review=review_path, | |
| family=item.get("family_key") or f"{family_backend}::{family_base}", | |
| rejected=item.get("rejected_count"), | |
| wins=item.get("total_wins"), | |
| losses=item.get("total_losses"), | |
| ties=item.get("total_ties"), | |
| ) | |
| ) | |
| return reasons | |
| def assert_recognizer_family_not_stopped(failure_review: str | Path, *, backend: str, base_model: str) -> None: | |
| reasons = stopped_recognizer_family_reasons(failure_review, backend=backend, base_model=base_model) | |
| if reasons: | |
| raise ValidationError("; ".join(reasons)) | |
| def _missing_requirements(payload: dict[str, Any]) -> list[str]: | |
| value = payload.get("missing_requirements") | |
| if not isinstance(value, list): | |
| return [] | |
| return [str(item) for item in value if isinstance(item, str) and item] | |
| def _failed_gates(payload: dict[str, Any]) -> list[FailedAdmissionGate]: | |
| gates = payload.get("gates") | |
| if not isinstance(gates, list): | |
| return [] | |
| failed: list[FailedAdmissionGate] = [] | |
| for item in gates: | |
| if not isinstance(item, dict) or item.get("passed") is True: | |
| continue | |
| gate = item.get("gate") | |
| gate_payload = gate if isinstance(gate, dict) else {} | |
| failed.append( | |
| FailedAdmissionGate( | |
| baseline=str(gate_payload.get("baseline") or ""), | |
| metric=str(gate_payload.get("metric") or ""), | |
| slices=_gate_slices(gate_payload), | |
| pairs=_int_or_none(item.get("pairs")), | |
| candidate_mean=_float_or_none(item.get("candidate_mean")), | |
| baseline_mean=_float_or_none(item.get("baseline_mean")), | |
| absolute_improvement=_float_or_none(item.get("absolute_improvement")), | |
| relative_improvement=_float_or_none(item.get("relative_improvement")), | |
| win_count=_int_or_none(item.get("win_count")), | |
| loss_count=_int_or_none(item.get("loss_count")), | |
| tie_count=_int_or_none(item.get("tie_count")), | |
| reason=str(item.get("reason")) if item.get("reason") else None, | |
| ) | |
| ) | |
| return failed | |
| def _gate_slices(gate: dict[str, Any]) -> list[str]: | |
| value = gate.get("slices") | |
| if isinstance(value, list): | |
| return [str(item) for item in value if str(item)] | |
| single = gate.get("slice") | |
| return [str(single)] if single else [] | |
| def _float_or_none(value: Any) -> float | None: | |
| if isinstance(value, bool) or not isinstance(value, int | float): | |
| return None | |
| as_float = float(value) | |
| return as_float if math.isfinite(as_float) else None | |
| def _int_or_none(value: Any) -> int | None: | |
| if isinstance(value, bool) or not isinstance(value, int | float): | |
| return None | |
| as_float = float(value) | |
| if not math.isfinite(as_float): | |
| return None | |
| return int(as_float) | |
| def _next_actions(decision: str, failed_gates: list[FailedAdmissionGate], missing_requirements: list[str]) -> list[str]: | |
| if decision == "admit": | |
| return [ | |
| "Attach this admission report to the model card and run claim preflight with --require-model-admission.", | |
| "Promote the recognizer only through the full document SOTA portfolio; admission is not a final SOTA claim.", | |
| ] | |
| actions: list[str] = [] | |
| failed_slices = {slice_name.lower() for gate in failed_gates for slice_name in gate.slices} | |
| failed_metrics = {gate.metric for gate in failed_gates} | |
| losses = sum((gate.loss_count or 0) for gate in failed_gates) | |
| wins = sum((gate.win_count or 0) for gate in failed_gates) | |
| if "english" in failed_slices or "en" in failed_slices or "line_crop" in failed_slices: | |
| actions.append("Do not train another Devanagari-only recognizer branch until English preservation is explicitly designed and gated.") | |
| if "nepali" in failed_slices: | |
| actions.append("Investigate the Nepali recognizer path separately; the current branch does not clear the Devanagari admission slice.") | |
| if "cer" in failed_metrics or "wer" in failed_metrics: | |
| actions.append("Prioritize recognizer correctness before document-structure integration; OCR gates are failing upstream.") | |
| if losses > 0 and wins == 0: | |
| actions.append("Stop this exact model family for engine promotion; it has no paired wins in the admission failure gates.") | |
| if not actions and missing_requirements: | |
| actions.append("Fix the missing validation requirements before spending more GPU time.") | |
| if not actions: | |
| actions.append("Treat this branch as rejected until a new validation report passes the admission gate.") | |
| return actions | |
| def _write_markdown(path: Path, decision: RecognizerAdmissionDecision) -> None: | |
| lines = [ | |
| "# Recognizer Admission Decision", | |
| "", | |
| f"Validation report: `{decision.validation_report}`", | |
| f"Decision: `{decision.decision}`", | |
| f"Claim status: `{decision.claim_status}`", | |
| f"Passed: `{'yes' if decision.passed else 'no'}`", | |
| f"Samples: `{decision.sample_count if decision.sample_count is not None else 'unknown'}`", | |
| "", | |
| "## Missing Requirements", | |
| "", | |
| ] | |
| lines.extend(f"- {item}" for item in decision.missing_requirements) if decision.missing_requirements else lines.append("- none") | |
| lines.extend( | |
| [ | |
| "", | |
| "## Failed Gates", | |
| "", | |
| "| baseline | metric | slices | pairs | candidate | baseline mean | abs improvement | wins | losses | ties | reason |", | |
| "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |", | |
| ] | |
| ) | |
| if decision.failed_gates: | |
| for gate in decision.failed_gates: | |
| lines.append( | |
| "| {baseline} | {metric} | {slices} | {pairs} | {candidate} | {baseline_mean} | {absolute} | {wins} | {losses} | {ties} | {reason} |".format( | |
| baseline=gate.baseline, | |
| metric=gate.metric, | |
| slices=",".join(gate.slices), | |
| pairs="" if gate.pairs is None else gate.pairs, | |
| candidate=_format_optional_float(gate.candidate_mean), | |
| baseline_mean=_format_optional_float(gate.baseline_mean), | |
| absolute=_format_optional_float(gate.absolute_improvement), | |
| wins="" if gate.win_count is None else gate.win_count, | |
| losses="" if gate.loss_count is None else gate.loss_count, | |
| ties="" if gate.tie_count is None else gate.tie_count, | |
| reason=(gate.reason or "").replace("|", "\\|"), | |
| ) | |
| ) | |
| else: | |
| lines.append("| none | | | | | | | | | | |") | |
| lines.extend(["", "## Next Actions", ""]) | |
| lines.extend(f"- {item}" for item in decision.next_actions) if decision.next_actions else lines.append("- none") | |
| path.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| def _read_json_object(path: Path, label: str) -> dict[str, Any]: | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except FileNotFoundError as exc: | |
| raise ValidationError(f"{label} does not exist: {path}") from exc | |
| except json.JSONDecodeError as exc: | |
| raise ValidationError(f"invalid {label} JSON {path}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise ValidationError(f"{label} must be a JSON object: {path}") | |
| return payload | |
| def _read_optional_validation_payload(decision_payload: dict[str, Any], decision_path: Path) -> dict[str, Any] | None: | |
| value = decision_payload.get("validation_report") | |
| if not isinstance(value, str) or not value: | |
| return None | |
| path = Path(value) | |
| if not path.is_absolute() and not path.exists(): | |
| path = decision_path.parent / path | |
| if not path.exists(): | |
| return None | |
| return _read_json_object(path, "admission validation report") | |
| def _decision_payload_from_validation(path: Path, payload: dict[str, Any]) -> dict[str, Any]: | |
| claim_status = str(payload.get("claim_status") or "") | |
| passed = bool(payload.get("passed")) | |
| return { | |
| "validation_report": str(path), | |
| "decision": "admit" if claim_status == "validated" and passed else "reject", | |
| "claim_status": claim_status or "not_run", | |
| "passed": passed, | |
| "sample_count": payload.get("sample_count"), | |
| "failed_gates": [gate.to_dict() for gate in _failed_gates(payload)], | |
| } | |
| def _model_identity_for_run( | |
| index: int, | |
| card_payloads: list[dict[str, Any]], | |
| cards: list[Path], | |
| validation_payload: dict[str, Any] | None, | |
| source_path: Path, | |
| ) -> tuple[str, str, str, str | None]: | |
| if index < len(card_payloads): | |
| payload = card_payloads[index] | |
| return ( | |
| str(payload.get("model_id") or "unknown"), | |
| str(payload.get("backend") or "unknown"), | |
| str(payload.get("base_model") or "unknown"), | |
| str(cards[index]), | |
| ) | |
| if validation_payload: | |
| candidate_path = _candidate_model_config_path(validation_payload, source_path) | |
| if candidate_path and candidate_path.exists(): | |
| payload = _read_json_object(candidate_path, "candidate model card") | |
| return ( | |
| str(payload.get("model_id") or "unknown"), | |
| str(payload.get("backend") or "unknown"), | |
| str(payload.get("base_model") or "unknown"), | |
| str(candidate_path), | |
| ) | |
| return ("unknown", "unknown", "unknown", None) | |
| def _candidate_model_config_path(validation_payload: dict[str, Any], source_path: Path) -> Path | None: | |
| provenance = validation_payload.get("provenance") | |
| if not isinstance(provenance, dict): | |
| return None | |
| candidate = provenance.get("candidate_model_config") | |
| if not isinstance(candidate, dict): | |
| return None | |
| path_value = candidate.get("path") | |
| if not isinstance(path_value, str) or not path_value: | |
| return None | |
| path = Path(path_value) | |
| if path.is_absolute() or path.exists(): | |
| return path | |
| relative = source_path.parent / path | |
| return relative if relative.exists() else path | |
| def _failure_run_from_decision(path: Path, payload: dict[str, Any], model_identity: tuple[str, str, str, str | None]) -> RecognizerFailureRun: | |
| gates = payload.get("failed_gates") | |
| failed_gates = [_failed_gate_from_payload(item) for item in gates if isinstance(item, dict)] if isinstance(gates, list) else [] | |
| model_id, backend, base_model, model_card = model_identity | |
| return RecognizerFailureRun( | |
| source_path=str(path), | |
| validation_report=str(payload.get("validation_report")) if payload.get("validation_report") else None, | |
| model_card=model_card, | |
| model_id=model_id, | |
| backend=backend, | |
| base_model=base_model, | |
| decision=str(payload.get("decision") or "unknown"), | |
| claim_status=str(payload.get("claim_status") or "not_run"), | |
| passed=bool(payload.get("passed")), | |
| sample_count=_int_or_none(payload.get("sample_count")), | |
| failed_gate_count=len(failed_gates), | |
| total_pairs=sum(gate.pairs or 0 for gate in failed_gates), | |
| total_wins=sum(gate.win_count or 0 for gate in failed_gates), | |
| total_losses=sum(gate.loss_count or 0 for gate in failed_gates), | |
| total_ties=sum(gate.tie_count or 0 for gate in failed_gates), | |
| failed_slices=sorted({slice_name for gate in failed_gates for slice_name in gate.slices}), | |
| failed_metrics=sorted({gate.metric for gate in failed_gates if gate.metric}), | |
| ) | |
| def _failed_gate_from_payload(payload: dict[str, Any]) -> FailedAdmissionGate: | |
| return FailedAdmissionGate( | |
| baseline=str(payload.get("baseline") or ""), | |
| metric=str(payload.get("metric") or ""), | |
| slices=[str(item) for item in payload.get("slices", []) if str(item)] if isinstance(payload.get("slices"), list) else [], | |
| pairs=_int_or_none(payload.get("pairs")), | |
| candidate_mean=_float_or_none(payload.get("candidate_mean")), | |
| baseline_mean=_float_or_none(payload.get("baseline_mean")), | |
| absolute_improvement=_float_or_none(payload.get("absolute_improvement")), | |
| relative_improvement=_float_or_none(payload.get("relative_improvement")), | |
| win_count=_int_or_none(payload.get("win_count")), | |
| loss_count=_int_or_none(payload.get("loss_count")), | |
| tie_count=_int_or_none(payload.get("tie_count")), | |
| reason=str(payload.get("reason")) if payload.get("reason") else None, | |
| ) | |
| def _failure_families(runs: list[RecognizerFailureRun], *, min_rejections_to_stop: int) -> list[RecognizerFailureFamily]: | |
| grouped: dict[tuple[str, str], list[RecognizerFailureRun]] = {} | |
| for run in runs: | |
| grouped.setdefault((run.backend, _canonical_recognizer_base_model(run.base_model)), []).append(run) | |
| families: list[RecognizerFailureFamily] = [] | |
| for (backend, base_model), family_runs in sorted(grouped.items()): | |
| rejected = [run for run in family_runs if run.decision == "reject"] | |
| admitted = [run for run in family_runs if run.decision == "admit"] | |
| total_wins = sum(run.total_wins for run in family_runs) | |
| total_losses = sum(run.total_losses for run in family_runs) | |
| recommendation = "continue_with_new_gate" | |
| rationale: list[str] = [] | |
| if len(rejected) >= min_rejections_to_stop and total_wins == 0 and total_losses > 0: | |
| recommendation = "stop_family" | |
| rationale.append(f"{len(rejected)} rejected runs meet the stop threshold of {min_rejections_to_stop}") | |
| rationale.append("failed admission gates have zero paired wins") | |
| elif rejected: | |
| recommendation = "do_not_promote" | |
| rationale.append(f"{len(rejected)} rejected runs remain unresolved") | |
| else: | |
| recommendation = "no_failure_guardrail" | |
| rationale.append("no rejected runs were provided for this family") | |
| families.append( | |
| RecognizerFailureFamily( | |
| family_key=f"{backend}::{base_model}", | |
| backend=backend, | |
| base_model=base_model, | |
| run_count=len(family_runs), | |
| rejected_count=len(rejected), | |
| admitted_count=len(admitted), | |
| total_failed_gates=sum(run.failed_gate_count for run in family_runs), | |
| total_pairs=sum(run.total_pairs for run in family_runs), | |
| total_wins=total_wins, | |
| total_losses=total_losses, | |
| total_ties=sum(run.total_ties for run in family_runs), | |
| failed_slices=sorted({slice_name for run in family_runs for slice_name in run.failed_slices}), | |
| failed_metrics=sorted({metric for run in family_runs for metric in run.failed_metrics}), | |
| recommendation=recommendation, | |
| rationale=rationale, | |
| ) | |
| ) | |
| return families | |
| def _canonical_recognizer_base_model(base_model: str) -> str: | |
| value = base_model.strip() | |
| lowered = value.lower() | |
| if "devanagari_pp-ocrv3_mobile_rec" in lowered: | |
| return "PaddlePaddle/devanagari_PP-OCRv3_mobile_rec" | |
| return value or "unknown" | |
| def _failure_review_next_actions(families: list[RecognizerFailureFamily]) -> list[str]: | |
| if any(family.recommendation == "stop_family" for family in families): | |
| return [ | |
| "Do not spend more GPU time on stopped recognizer families without a different base architecture or training objective.", | |
| "Move engine work toward measurable document parsing with admitted recognizer components or a document-native model.", | |
| "Keep these negative runs in the paper as ablation evidence, not as promoted model results.", | |
| ] | |
| if any(family.recommendation == "do_not_promote" for family in families): | |
| return [ | |
| "Do not promote rejected recognizers into the engine until a new admission report passes.", | |
| "Add more runs or model-card bindings before declaring a family stopped.", | |
| ] | |
| return ["No rejected recognizer family was strong enough to trigger a stop recommendation."] | |
| def _write_failure_review_markdown(path: Path, review: RecognizerFailureReview) -> None: | |
| lines = [ | |
| "# Recognizer Failure Review", | |
| "", | |
| f"Admission decisions: `{len(review.admission_decisions)}`", | |
| f"Validation reports: `{len(review.validation_reports)}`", | |
| f"Model cards: `{len(review.model_cards)}`", | |
| f"Stop threshold: `{review.min_rejections_to_stop}` rejected runs with zero paired wins", | |
| "", | |
| "## Families", | |
| "", | |
| "| family | runs | rejected | admitted | failed gates | pairs | wins | losses | slices | recommendation |", | |
| "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |", | |
| ] | |
| for family in review.families: | |
| lines.append( | |
| "| {family} | {runs} | {rejected} | {admitted} | {failed_gates} | {pairs} | {wins} | {losses} | {slices} | {recommendation} |".format( | |
| family=family.family_key.replace("|", "\\|"), | |
| runs=family.run_count, | |
| rejected=family.rejected_count, | |
| admitted=family.admitted_count, | |
| failed_gates=family.total_failed_gates, | |
| pairs=family.total_pairs, | |
| wins=family.total_wins, | |
| losses=family.total_losses, | |
| slices=",".join(family.failed_slices), | |
| recommendation=family.recommendation, | |
| ) | |
| ) | |
| lines.extend(["", "## Runs", ""]) | |
| for run in review.runs: | |
| family = f"{run.backend}::{_canonical_recognizer_base_model(run.base_model)}" | |
| lines.append( | |
| f"- `{run.source_path}`: decision=`{run.decision}` model=`{run.model_id}` family=`{family}` wins/losses/ties=`{run.total_wins}/{run.total_losses}/{run.total_ties}`" | |
| ) | |
| lines.extend(["", "## Next Actions", ""]) | |
| lines.extend(f"- {item}" for item in review.next_actions) | |
| path.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| def _format_optional_float(value: float | None) -> str: | |
| return "" if value is None else f"{value:.6f}" | |