from __future__ import annotations import hashlib import importlib.util import json import sys from argparse import Namespace from pathlib import Path def _load_tool(): path = Path(__file__).resolve().parents[1] / "tools" / "audit_recognizer_training_completion.py" spec = importlib.util.spec_from_file_location("audit_recognizer_training_completion", path) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def _write_json(path: Path, payload: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") def _fixture(tmp_path: Path, *, checkpoint: bool = True, sealed_used: bool = False) -> Path: workspace = tmp_path / "job" / "workspace" recipe = workspace / "training-recipes" / "fixture" checkpoint_dir = recipe / "checkpoints" checkpoint_dir.mkdir(parents=True) (recipe / "paddle_recognizer.yml").write_text( "Global:\n" " character_dict_path: training-recipes/fixture/nep_eng_dict.txt\n" " save_model_dir: training-recipes/fixture/checkpoints\n", encoding="utf-8", ) (recipe / "nep_eng_dict.txt").write_text("क\nA\n", encoding="utf-8") (recipe / "recognizer-training-summary.json").write_text('{"backend":"paddleocr"}\n', encoding="utf-8") (checkpoint_dir / "train.log").write_text("best_accuracy\n", encoding="utf-8") if checkpoint: (checkpoint_dir / "best_accuracy.pdparams").write_bytes(b"checkpoint") _write_json( tmp_path / "recipes.json", { "recipes": [ { "id": "fixture-heldout", "task": "recognizer_oracle_crop", "language": "limbu", "script": "Devanagari", "model_family": "fixture-model", "runner": { "type": "external_command", "command_template": "REPLACE_WITH --labels {label_file}", "requires_operator_resolution": True, "selected_checkpoint_on_test": False, }, "eval_pack": { "label_file": "eval.txt", "data_dir": "eval-pack", "manifest_file": "manifests/eval.jsonl", "summary_file": "summaries/eval.summary.json", "script_inventory_file": "script-inventory.json", "domain": "real_print", "gt_status": "curriculum_text_unreviewed", "split": "eval", "eval_pack_version": "fixture-v1", "frozen_eval_pack": False, "unseen_for_selection": True, "claim_eligible": False, }, "leakage_sources": [{"manifest_file": "manifests/train-dev.jsonl", "split": "train_dev"}], "claim_blockers": [ "trained checkpoint path and reproducible prediction command are unresolved until training", "eval pack is not frozen", "packaged local model card is absent", ], } ] }, ) _write_json( tmp_path / "plan.json", { "plans": [ { "id": "fixture-plan", "model_family": "fixture-model", "training_bundle": { "archive": "bundle.tar.gz", "base_model": "PP-OCRv5_mobile_rec_pretrained.pdparams", }, "selection_policy": { "train_manifest": "manifests/train.jsonl", "dev_manifest": "manifests/dev.jsonl", "sealed_eval_manifest": "manifests/eval.jsonl", "sealed_eval_label_file": "eval.txt", }, "admission_recipe": { "registry": "recipes.json", "recipe_id": "fixture-heldout", }, } ] }, ) job = { "validation": {"training_ready": True, "heldout_admission_ready": False}, "plan": { "path": str(tmp_path / "plan.json"), "id": "fixture-plan", "model_family": "fixture-model", }, "training_bundle": { "archive": str(tmp_path / "bundle.tar.gz"), "config_member": "training-recipes/fixture/paddle_recognizer.yml", }, "sealed_eval": { "manifest": str(tmp_path / "manifests" / "eval.jsonl"), "label_file": str(tmp_path / "eval.txt"), "used_for_training": sealed_used, "archive_member_hits": [], }, "external_dependencies": { "paddleocr_dir": "/tmp/PaddleOCR-v3.7.0", }, "outputs": { "workspace_default": str(workspace), }, } job_path = tmp_path / "job" / "recognizer-training-job.json" _write_json(job_path, job) return job_path def test_audit_recognizer_training_completion_writes_post_training_handoff(tmp_path: Path) -> None: tool = _load_tool() job_path = _fixture(tmp_path) report = tool.audit_completion( Namespace( job_manifest=job_path, workspace=None, root=tmp_path, out=tmp_path / "handoff", checkpoint_name="best_accuracy", checkpoint_dir=None, dictionary=None, model_id="fixture-trained", ) ) assert report["ready_for_export"] is True assert report["issue_count"] == 0 assert report["checkpoint"]["files"]["pdparams"]["sha256"] == _sha256( tmp_path / "job" / "workspace" / "training-recipes" / "fixture" / "checkpoints" / "best_accuracy.pdparams" ) commands = report["commands"] assert "export-recognizer" in commands["export_command"] assert "package-recognizer" in commands["package_command"] assert "predict_recognizer_eval.py" in commands["predict_sealed_eval_command"] assert "run_eval_recipe.py" in commands["run_resolved_recipe_command"] resolved = json.loads((tmp_path / "handoff" / "resolved-heldout-eval-recipe.json").read_text(encoding="utf-8")) runner = resolved["recipes"][0]["runner"] assert runner["type"] == "model_config" assert runner["selected_checkpoint_on_test"] is False assert "trained checkpoint path" not in " ".join(resolved["recipes"][0]["claim_blockers"]) def test_audit_recognizer_training_completion_rejects_missing_checkpoint(tmp_path: Path) -> None: tool = _load_tool() job_path = _fixture(tmp_path, checkpoint=False) report = tool.audit_completion( Namespace( job_manifest=job_path, workspace=None, root=tmp_path, out=tmp_path / "handoff", checkpoint_name="best_accuracy", checkpoint_dir=None, dictionary=None, model_id=None, ) ) assert report["ready_for_export"] is False assert any("selected checkpoint pdparams does not exist" in issue for issue in report["issues"]) assert report["resolved_recipe"] is None def test_audit_recognizer_training_completion_rejects_sealed_eval_training_use(tmp_path: Path) -> None: tool = _load_tool() job_path = _fixture(tmp_path, sealed_used=True) report = tool.audit_completion( Namespace( job_manifest=job_path, workspace=None, root=tmp_path, out=tmp_path / "handoff", checkpoint_name="best_accuracy", checkpoint_dir=None, dictionary=None, model_id=None, ) ) assert report["ready_for_export"] is False assert any("sealed eval was excluded" in issue for issue in report["issues"])