ocr-projects-archive / ocr-tech-mps-integration /tests /test_audit_recognizer_training_completion.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
8.32 kB
from __future__ import annotations
import hashlib
import importlib.util
import json
import sys
from argparse import Namespace
from pathlib import Path
def _load_tool():
path = Path(__file__).resolve().parents[1] / "tools" / "audit_recognizer_training_completion.py"
spec = importlib.util.spec_from_file_location("audit_recognizer_training_completion", path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _write_json(path: Path, payload: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8")
def _fixture(tmp_path: Path, *, checkpoint: bool = True, sealed_used: bool = False) -> Path:
workspace = tmp_path / "job" / "workspace"
recipe = workspace / "training-recipes" / "fixture"
checkpoint_dir = recipe / "checkpoints"
checkpoint_dir.mkdir(parents=True)
(recipe / "paddle_recognizer.yml").write_text(
"Global:\n"
" character_dict_path: training-recipes/fixture/nep_eng_dict.txt\n"
" save_model_dir: training-recipes/fixture/checkpoints\n",
encoding="utf-8",
)
(recipe / "nep_eng_dict.txt").write_text("क\nA\n", encoding="utf-8")
(recipe / "recognizer-training-summary.json").write_text('{"backend":"paddleocr"}\n', encoding="utf-8")
(checkpoint_dir / "train.log").write_text("best_accuracy\n", encoding="utf-8")
if checkpoint:
(checkpoint_dir / "best_accuracy.pdparams").write_bytes(b"checkpoint")
_write_json(
tmp_path / "recipes.json",
{
"recipes": [
{
"id": "fixture-heldout",
"task": "recognizer_oracle_crop",
"language": "limbu",
"script": "Devanagari",
"model_family": "fixture-model",
"runner": {
"type": "external_command",
"command_template": "REPLACE_WITH --labels {label_file}",
"requires_operator_resolution": True,
"selected_checkpoint_on_test": False,
},
"eval_pack": {
"label_file": "eval.txt",
"data_dir": "eval-pack",
"manifest_file": "manifests/eval.jsonl",
"summary_file": "summaries/eval.summary.json",
"script_inventory_file": "script-inventory.json",
"domain": "real_print",
"gt_status": "curriculum_text_unreviewed",
"split": "eval",
"eval_pack_version": "fixture-v1",
"frozen_eval_pack": False,
"unseen_for_selection": True,
"claim_eligible": False,
},
"leakage_sources": [{"manifest_file": "manifests/train-dev.jsonl", "split": "train_dev"}],
"claim_blockers": [
"trained checkpoint path and reproducible prediction command are unresolved until training",
"eval pack is not frozen",
"packaged local model card is absent",
],
}
]
},
)
_write_json(
tmp_path / "plan.json",
{
"plans": [
{
"id": "fixture-plan",
"model_family": "fixture-model",
"training_bundle": {
"archive": "bundle.tar.gz",
"base_model": "PP-OCRv5_mobile_rec_pretrained.pdparams",
},
"selection_policy": {
"train_manifest": "manifests/train.jsonl",
"dev_manifest": "manifests/dev.jsonl",
"sealed_eval_manifest": "manifests/eval.jsonl",
"sealed_eval_label_file": "eval.txt",
},
"admission_recipe": {
"registry": "recipes.json",
"recipe_id": "fixture-heldout",
},
}
]
},
)
job = {
"validation": {"training_ready": True, "heldout_admission_ready": False},
"plan": {
"path": str(tmp_path / "plan.json"),
"id": "fixture-plan",
"model_family": "fixture-model",
},
"training_bundle": {
"archive": str(tmp_path / "bundle.tar.gz"),
"config_member": "training-recipes/fixture/paddle_recognizer.yml",
},
"sealed_eval": {
"manifest": str(tmp_path / "manifests" / "eval.jsonl"),
"label_file": str(tmp_path / "eval.txt"),
"used_for_training": sealed_used,
"archive_member_hits": [],
},
"external_dependencies": {
"paddleocr_dir": "/tmp/PaddleOCR-v3.7.0",
},
"outputs": {
"workspace_default": str(workspace),
},
}
job_path = tmp_path / "job" / "recognizer-training-job.json"
_write_json(job_path, job)
return job_path
def test_audit_recognizer_training_completion_writes_post_training_handoff(tmp_path: Path) -> None:
tool = _load_tool()
job_path = _fixture(tmp_path)
report = tool.audit_completion(
Namespace(
job_manifest=job_path,
workspace=None,
root=tmp_path,
out=tmp_path / "handoff",
checkpoint_name="best_accuracy",
checkpoint_dir=None,
dictionary=None,
model_id="fixture-trained",
)
)
assert report["ready_for_export"] is True
assert report["issue_count"] == 0
assert report["checkpoint"]["files"]["pdparams"]["sha256"] == _sha256(
tmp_path / "job" / "workspace" / "training-recipes" / "fixture" / "checkpoints" / "best_accuracy.pdparams"
)
commands = report["commands"]
assert "export-recognizer" in commands["export_command"]
assert "package-recognizer" in commands["package_command"]
assert "predict_recognizer_eval.py" in commands["predict_sealed_eval_command"]
assert "run_eval_recipe.py" in commands["run_resolved_recipe_command"]
resolved = json.loads((tmp_path / "handoff" / "resolved-heldout-eval-recipe.json").read_text(encoding="utf-8"))
runner = resolved["recipes"][0]["runner"]
assert runner["type"] == "model_config"
assert runner["selected_checkpoint_on_test"] is False
assert "trained checkpoint path" not in " ".join(resolved["recipes"][0]["claim_blockers"])
def test_audit_recognizer_training_completion_rejects_missing_checkpoint(tmp_path: Path) -> None:
tool = _load_tool()
job_path = _fixture(tmp_path, checkpoint=False)
report = tool.audit_completion(
Namespace(
job_manifest=job_path,
workspace=None,
root=tmp_path,
out=tmp_path / "handoff",
checkpoint_name="best_accuracy",
checkpoint_dir=None,
dictionary=None,
model_id=None,
)
)
assert report["ready_for_export"] is False
assert any("selected checkpoint pdparams does not exist" in issue for issue in report["issues"])
assert report["resolved_recipe"] is None
def test_audit_recognizer_training_completion_rejects_sealed_eval_training_use(tmp_path: Path) -> None:
tool = _load_tool()
job_path = _fixture(tmp_path, sealed_used=True)
report = tool.audit_completion(
Namespace(
job_manifest=job_path,
workspace=None,
root=tmp_path,
out=tmp_path / "handoff",
checkpoint_name="best_accuracy",
checkpoint_dir=None,
dictionary=None,
model_id=None,
)
)
assert report["ready_for_export"] is False
assert any("sealed eval was excluded" in issue for issue in report["issues"])