Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_language_registry.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 6.84 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_language_registry.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_language_registry.py
-
curl -L -o test_language_registry.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_language_registry.py
6.84 kB
| from __future__ import annotations | |
| import json | |
| import struct | |
| from copy import deepcopy | |
| from pathlib import Path | |
| from ocrtech.cli import main | |
| from ocrtech.language_registry import audit_font_renderability, audit_language_registry | |
| ROOT = Path(__file__).resolve().parents[1] | |
| REGISTRY = ROOT / "languages" / "nepal-v1.json" | |
| def test_nepal_language_registry_passes_core_gates(tmp_path: Path) -> None: | |
| audit = audit_language_registry(REGISTRY, tmp_path / "audit") | |
| assert audit.passed | |
| assert audit.language_count >= 11 | |
| assert audit.counted_nepal_language_count >= 10 | |
| assert audit.first_target_ids == ["limbu"] | |
| assert audit.primary_scripts["limbu"] == ["Limbu / Sirijonga"] | |
| assert "english" in audit.primary_scripts | |
| assert not audit.errors | |
| assert (tmp_path / "audit" / "language-registry-audit.json").exists() | |
| assert (tmp_path / "audit" / "language-registry-audit.md").exists() | |
| def test_language_registry_rejects_non_limbu_first(tmp_path: Path) -> None: | |
| payload = json.loads(REGISTRY.read_text(encoding="utf-8")) | |
| payload["languages"] = [payload["languages"][1], payload["languages"][0], *payload["languages"][2:]] | |
| registry = tmp_path / "bad-order.json" | |
| registry.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") | |
| audit = audit_language_registry(registry, tmp_path / "audit") | |
| assert not audit.passed | |
| assert "first language entry must have id 'limbu'" in audit.errors | |
| def test_language_registry_rejects_too_few_counted_nepal_languages(tmp_path: Path) -> None: | |
| payload = json.loads(REGISTRY.read_text(encoding="utf-8")) | |
| reduced = deepcopy(payload) | |
| counted = 0 | |
| for language in reduced["languages"]: | |
| if language["counts_toward_nepal_language_target"]: | |
| counted += 1 | |
| if counted > 3: | |
| language["counts_toward_nepal_language_target"] = False | |
| registry = tmp_path / "too-few.json" | |
| registry.write_text(json.dumps(reduced, ensure_ascii=False), encoding="utf-8") | |
| audit = audit_language_registry(registry, tmp_path / "audit", min_nepal_languages=10) | |
| assert not audit.passed | |
| assert any(error.startswith("registry counts 3 Nepal languages") for error in audit.errors) | |
| def test_language_registry_rejects_english_counting_toward_target(tmp_path: Path) -> None: | |
| payload = json.loads(REGISTRY.read_text(encoding="utf-8")) | |
| for language in payload["languages"]: | |
| if language["id"] == "english": | |
| language["counts_toward_nepal_language_target"] = True | |
| registry = tmp_path / "english-counted.json" | |
| registry.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") | |
| audit = audit_language_registry(registry, tmp_path / "audit") | |
| assert not audit.passed | |
| assert "english must not count toward the Nepal-language target" in audit.errors | |
| def test_cli_audit_language_registry_writes_summary(tmp_path: Path, capsys) -> None: | |
| exit_code = main(["audit-language-registry", str(REGISTRY), "--out", str(tmp_path / "audit")]) | |
| captured = capsys.readouterr() | |
| payload = json.loads((tmp_path / "audit" / "language-registry-audit.json").read_text(encoding="utf-8")) | |
| assert exit_code == 0 | |
| assert "passed=True" in captured.out | |
| assert "first_targets=limbu" in captured.out | |
| assert payload["counted_nepal_language_count"] >= 10 | |
| def test_font_renderability_audit_scores_cmap_coverage(tmp_path: Path) -> None: | |
| font = tmp_path / "single-a.ttf" | |
| _write_minimal_cmap_font(font, 0x0041) | |
| registry = tmp_path / "registry.json" | |
| registry.write_text( | |
| json.dumps( | |
| { | |
| "languages": [ | |
| { | |
| "id": "fixture", | |
| "scripts": [ | |
| { | |
| "script": "Latin", | |
| "primary": True, | |
| "unicode_ranges": ["U+0041"], | |
| } | |
| ], | |
| "font_candidates": [ | |
| {"name": "Fixture", "path": str(font), "status": "pending_render_audit"} | |
| ], | |
| } | |
| ] | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| audit = audit_font_renderability(registry, tmp_path / "audit") | |
| assert audit.passed | |
| assert audit.script_count == 1 | |
| assert audit.candidates[0].coverage_ratio == 1.0 | |
| assert (tmp_path / "audit" / "font-renderability-audit.json").exists() | |
| assert (tmp_path / "audit" / "font-renderability-audit.md").exists() | |
| def test_cli_font_renderability_audit_fails_missing_coverage(tmp_path: Path, capsys) -> None: | |
| font = tmp_path / "single-a.ttf" | |
| _write_minimal_cmap_font(font, 0x0041) | |
| registry = tmp_path / "registry.json" | |
| registry.write_text( | |
| json.dumps( | |
| { | |
| "languages": [ | |
| { | |
| "id": "fixture", | |
| "scripts": [ | |
| { | |
| "script": "Latin", | |
| "primary": True, | |
| "unicode_ranges": ["U+0042"], | |
| } | |
| ], | |
| "font_candidates": [ | |
| {"name": "Fixture", "path": str(font), "status": "pending_render_audit"} | |
| ], | |
| } | |
| ] | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| exit_code = main(["audit-font-renderability", str(registry), "--out", str(tmp_path / "audit")]) | |
| captured = capsys.readouterr() | |
| payload = json.loads((tmp_path / "audit" / "font-renderability-audit.json").read_text(encoding="utf-8")) | |
| assert exit_code == 1 | |
| assert "passed=False" in captured.out | |
| assert payload["errors"] == ["fixture:Latin: no font candidate covers at least 95.00% of required codepoints"] | |
| assert payload["candidates"][0]["missing_sample"] == ["U+0042"] | |
| def _write_minimal_cmap_font(path: Path, codepoint: int) -> None: | |
| seg_count = 2 | |
| end_codes = [codepoint, 0xFFFF] | |
| start_codes = [codepoint, 0xFFFF] | |
| id_deltas = [1, 1] | |
| id_range_offsets = [0, 0] | |
| subtable = b"".join( | |
| [ | |
| struct.pack(">HHHHHHH", 4, 32, 0, seg_count * 2, 0, 0, 0), | |
| struct.pack(">HH", *end_codes), | |
| struct.pack(">H", 0), | |
| struct.pack(">HH", *start_codes), | |
| struct.pack(">hh", *id_deltas), | |
| struct.pack(">HH", *id_range_offsets), | |
| ] | |
| ) | |
| cmap = struct.pack(">HHHHL", 0, 1, 3, 1, 12) + subtable | |
| offset_table = struct.pack(">LHHHH", 0x00010000, 1, 0, 0, 0) | |
| table_record = b"cmap" + struct.pack(">LLL", 0, len(offset_table) + 16, len(cmap)) | |
| path.write_bytes(offset_table + table_record + cmap) | |