ocr-projects-archive / ocr-tech-mps-integration /tests /test_audit_recognizer_training_completion.py
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/tests/test_audit_recognizer_training_completion.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 8.32 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/tests/test_audit_recognizer_training_completion.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/tests/test_audit_recognizer_training_completion.py
-
curl -L -o test_audit_recognizer_training_completion.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/tests/test_audit_recognizer_training_completion.py
8.32 kB
| from __future__ import annotations | |
| import hashlib | |
| import importlib.util | |
| import json | |
| import sys | |
| from argparse import Namespace | |
| from pathlib import Path | |
| def _load_tool(): | |
| path = Path(__file__).resolve().parents[1] / "tools" / "audit_recognizer_training_completion.py" | |
| spec = importlib.util.spec_from_file_location("audit_recognizer_training_completion", path) | |
| assert spec is not None | |
| assert spec.loader is not None | |
| module = importlib.util.module_from_spec(spec) | |
| sys.modules[spec.name] = module | |
| spec.loader.exec_module(module) | |
| return module | |
| def _sha256(path: Path) -> str: | |
| return hashlib.sha256(path.read_bytes()).hexdigest() | |
| def _write_json(path: Path, payload: object) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") | |
| def _fixture(tmp_path: Path, *, checkpoint: bool = True, sealed_used: bool = False) -> Path: | |
| workspace = tmp_path / "job" / "workspace" | |
| recipe = workspace / "training-recipes" / "fixture" | |
| checkpoint_dir = recipe / "checkpoints" | |
| checkpoint_dir.mkdir(parents=True) | |
| (recipe / "paddle_recognizer.yml").write_text( | |
| "Global:\n" | |
| " character_dict_path: training-recipes/fixture/nep_eng_dict.txt\n" | |
| " save_model_dir: training-recipes/fixture/checkpoints\n", | |
| encoding="utf-8", | |
| ) | |
| (recipe / "nep_eng_dict.txt").write_text("क\nA\n", encoding="utf-8") | |
| (recipe / "recognizer-training-summary.json").write_text('{"backend":"paddleocr"}\n', encoding="utf-8") | |
| (checkpoint_dir / "train.log").write_text("best_accuracy\n", encoding="utf-8") | |
| if checkpoint: | |
| (checkpoint_dir / "best_accuracy.pdparams").write_bytes(b"checkpoint") | |
| _write_json( | |
| tmp_path / "recipes.json", | |
| { | |
| "recipes": [ | |
| { | |
| "id": "fixture-heldout", | |
| "task": "recognizer_oracle_crop", | |
| "language": "limbu", | |
| "script": "Devanagari", | |
| "model_family": "fixture-model", | |
| "runner": { | |
| "type": "external_command", | |
| "command_template": "REPLACE_WITH --labels {label_file}", | |
| "requires_operator_resolution": True, | |
| "selected_checkpoint_on_test": False, | |
| }, | |
| "eval_pack": { | |
| "label_file": "eval.txt", | |
| "data_dir": "eval-pack", | |
| "manifest_file": "manifests/eval.jsonl", | |
| "summary_file": "summaries/eval.summary.json", | |
| "script_inventory_file": "script-inventory.json", | |
| "domain": "real_print", | |
| "gt_status": "curriculum_text_unreviewed", | |
| "split": "eval", | |
| "eval_pack_version": "fixture-v1", | |
| "frozen_eval_pack": False, | |
| "unseen_for_selection": True, | |
| "claim_eligible": False, | |
| }, | |
| "leakage_sources": [{"manifest_file": "manifests/train-dev.jsonl", "split": "train_dev"}], | |
| "claim_blockers": [ | |
| "trained checkpoint path and reproducible prediction command are unresolved until training", | |
| "eval pack is not frozen", | |
| "packaged local model card is absent", | |
| ], | |
| } | |
| ] | |
| }, | |
| ) | |
| _write_json( | |
| tmp_path / "plan.json", | |
| { | |
| "plans": [ | |
| { | |
| "id": "fixture-plan", | |
| "model_family": "fixture-model", | |
| "training_bundle": { | |
| "archive": "bundle.tar.gz", | |
| "base_model": "PP-OCRv5_mobile_rec_pretrained.pdparams", | |
| }, | |
| "selection_policy": { | |
| "train_manifest": "manifests/train.jsonl", | |
| "dev_manifest": "manifests/dev.jsonl", | |
| "sealed_eval_manifest": "manifests/eval.jsonl", | |
| "sealed_eval_label_file": "eval.txt", | |
| }, | |
| "admission_recipe": { | |
| "registry": "recipes.json", | |
| "recipe_id": "fixture-heldout", | |
| }, | |
| } | |
| ] | |
| }, | |
| ) | |
| job = { | |
| "validation": {"training_ready": True, "heldout_admission_ready": False}, | |
| "plan": { | |
| "path": str(tmp_path / "plan.json"), | |
| "id": "fixture-plan", | |
| "model_family": "fixture-model", | |
| }, | |
| "training_bundle": { | |
| "archive": str(tmp_path / "bundle.tar.gz"), | |
| "config_member": "training-recipes/fixture/paddle_recognizer.yml", | |
| }, | |
| "sealed_eval": { | |
| "manifest": str(tmp_path / "manifests" / "eval.jsonl"), | |
| "label_file": str(tmp_path / "eval.txt"), | |
| "used_for_training": sealed_used, | |
| "archive_member_hits": [], | |
| }, | |
| "external_dependencies": { | |
| "paddleocr_dir": "/tmp/PaddleOCR-v3.7.0", | |
| }, | |
| "outputs": { | |
| "workspace_default": str(workspace), | |
| }, | |
| } | |
| job_path = tmp_path / "job" / "recognizer-training-job.json" | |
| _write_json(job_path, job) | |
| return job_path | |
| def test_audit_recognizer_training_completion_writes_post_training_handoff(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| job_path = _fixture(tmp_path) | |
| report = tool.audit_completion( | |
| Namespace( | |
| job_manifest=job_path, | |
| workspace=None, | |
| root=tmp_path, | |
| out=tmp_path / "handoff", | |
| checkpoint_name="best_accuracy", | |
| checkpoint_dir=None, | |
| dictionary=None, | |
| model_id="fixture-trained", | |
| ) | |
| ) | |
| assert report["ready_for_export"] is True | |
| assert report["issue_count"] == 0 | |
| assert report["checkpoint"]["files"]["pdparams"]["sha256"] == _sha256( | |
| tmp_path / "job" / "workspace" / "training-recipes" / "fixture" / "checkpoints" / "best_accuracy.pdparams" | |
| ) | |
| commands = report["commands"] | |
| assert "export-recognizer" in commands["export_command"] | |
| assert "package-recognizer" in commands["package_command"] | |
| assert "predict_recognizer_eval.py" in commands["predict_sealed_eval_command"] | |
| assert "run_eval_recipe.py" in commands["run_resolved_recipe_command"] | |
| resolved = json.loads((tmp_path / "handoff" / "resolved-heldout-eval-recipe.json").read_text(encoding="utf-8")) | |
| runner = resolved["recipes"][0]["runner"] | |
| assert runner["type"] == "model_config" | |
| assert runner["selected_checkpoint_on_test"] is False | |
| assert "trained checkpoint path" not in " ".join(resolved["recipes"][0]["claim_blockers"]) | |
| def test_audit_recognizer_training_completion_rejects_missing_checkpoint(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| job_path = _fixture(tmp_path, checkpoint=False) | |
| report = tool.audit_completion( | |
| Namespace( | |
| job_manifest=job_path, | |
| workspace=None, | |
| root=tmp_path, | |
| out=tmp_path / "handoff", | |
| checkpoint_name="best_accuracy", | |
| checkpoint_dir=None, | |
| dictionary=None, | |
| model_id=None, | |
| ) | |
| ) | |
| assert report["ready_for_export"] is False | |
| assert any("selected checkpoint pdparams does not exist" in issue for issue in report["issues"]) | |
| assert report["resolved_recipe"] is None | |
| def test_audit_recognizer_training_completion_rejects_sealed_eval_training_use(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| job_path = _fixture(tmp_path, sealed_used=True) | |
| report = tool.audit_completion( | |
| Namespace( | |
| job_manifest=job_path, | |
| workspace=None, | |
| root=tmp_path, | |
| out=tmp_path / "handoff", | |
| checkpoint_name="best_accuracy", | |
| checkpoint_dir=None, | |
| dictionary=None, | |
| model_id=None, | |
| ) | |
| ) | |
| assert report["ready_for_export"] is False | |
| assert any("sealed eval was excluded" in issue for issue in report["issues"]) | |