Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download src/ocrtech/language_registry.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 33 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/src/ocrtech/language_registry.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/src/ocrtech/language_registry.py
-
curl -L -o language_registry.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/src/ocrtech/language_registry.py
33 kB
| """Audits for multilingual OCR language registries.""" | |
| from __future__ import annotations | |
| import getpass | |
| import json | |
| import re | |
| import struct | |
| import unicodedata | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import Any | |
| from .errors import DataValidationError | |
| UNICODE_RANGE_RE = re.compile(r"^U\+[0-9A-F]{4,6}(?:\.\.U\+[0-9A-F]{4,6})?$") | |
| LANGUAGE_REQUIRED_FIELDS = ( | |
| "id", | |
| "name", | |
| "priority", | |
| "counts_toward_nepal_language_target", | |
| "first_target", | |
| "scripts", | |
| "font_candidates", | |
| "paddle_dictionary", | |
| "normalization", | |
| "corpora", | |
| "validation_sources", | |
| "gorkhapatra_language_labels", | |
| ) | |
| SCRIPT_REQUIRED_FIELDS = ("script", "primary", "status", "unicode_ranges") | |
| FONT_PATH_STATUSES = {"pending_render_audit", "exists_local", "schema_audit_pending"} | |
| class FontCandidateAudit: | |
| language_id: str | |
| script: str | |
| font_name: str | |
| font_path: str | |
| status: str | |
| path_exists: bool | |
| supported_font_file: bool | |
| required_codepoints: int | |
| covered_codepoints: int | |
| coverage_ratio: float | |
| missing_sample: list[str] = field(default_factory=list) | |
| render_sample_path: str | None = None | |
| render_status: str = "not_requested" | |
| error: str | None = None | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "language_id": self.language_id, | |
| "script": self.script, | |
| "font_name": self.font_name, | |
| "font_path": self.font_path, | |
| "status": self.status, | |
| "path_exists": self.path_exists, | |
| "supported_font_file": self.supported_font_file, | |
| "required_codepoints": self.required_codepoints, | |
| "covered_codepoints": self.covered_codepoints, | |
| "coverage_ratio": self.coverage_ratio, | |
| "missing_sample": self.missing_sample, | |
| "render_sample_path": self.render_sample_path, | |
| "render_status": self.render_status, | |
| "error": self.error, | |
| } | |
| class FontRenderabilityAudit: | |
| registry_path: str | |
| passed: bool | |
| language_count: int | |
| script_count: int | |
| candidate_count: int | |
| render_requested: bool | |
| pillow_available: bool | |
| passing_scripts: list[str] = field(default_factory=list) | |
| failing_scripts: list[str] = field(default_factory=list) | |
| candidates: list[FontCandidateAudit] = field(default_factory=list) | |
| warnings: list[str] = field(default_factory=list) | |
| errors: list[str] = field(default_factory=list) | |
| summary_json_path: str | None = None | |
| summary_md_path: str | None = None | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "registry_path": self.registry_path, | |
| "passed": self.passed, | |
| "language_count": self.language_count, | |
| "script_count": self.script_count, | |
| "candidate_count": self.candidate_count, | |
| "render_requested": self.render_requested, | |
| "pillow_available": self.pillow_available, | |
| "passing_scripts": self.passing_scripts, | |
| "failing_scripts": self.failing_scripts, | |
| "warnings": self.warnings, | |
| "errors": self.errors, | |
| "candidates": [candidate.to_dict() for candidate in self.candidates], | |
| } | |
| class LanguageRegistryAudit: | |
| registry_path: str | |
| registry_id: str | |
| passed: bool | |
| language_count: int | |
| counted_nepal_language_count: int | |
| first_target_ids: list[str] | |
| primary_scripts: dict[str, list[str]] | |
| missing_local_paths: list[str] = field(default_factory=list) | |
| warnings: list[str] = field(default_factory=list) | |
| errors: list[str] = field(default_factory=list) | |
| summary_json_path: str | None = None | |
| summary_md_path: str | None = None | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "registry_path": self.registry_path, | |
| "registry_id": self.registry_id, | |
| "passed": self.passed, | |
| "language_count": self.language_count, | |
| "counted_nepal_language_count": self.counted_nepal_language_count, | |
| "first_target_ids": self.first_target_ids, | |
| "primary_scripts": self.primary_scripts, | |
| "missing_local_paths": self.missing_local_paths, | |
| "warnings": self.warnings, | |
| "errors": self.errors, | |
| } | |
| def audit_language_registry( | |
| registry_path: str | Path, | |
| out_dir: str | Path | None = None, | |
| *, | |
| min_nepal_languages: int = 10, | |
| require_limbu_first: bool = True, | |
| verify_local_paths: bool = True, | |
| ) -> LanguageRegistryAudit: | |
| path = Path(registry_path) | |
| payload = _load_json_object(path) | |
| languages = payload.get("languages") | |
| if not isinstance(languages, list) or not languages: | |
| raise DataValidationError(f"language registry must contain a non-empty languages list: {path}") | |
| errors: list[str] = [] | |
| warnings: list[str] = [] | |
| missing_local_paths: list[str] = [] | |
| ids: set[str] = set() | |
| priorities: set[int] = set() | |
| counted_ids: list[str] = [] | |
| first_target_ids: list[str] = [] | |
| primary_scripts: dict[str, list[str]] = {} | |
| for index, language in enumerate(languages): | |
| prefix = f"languages[{index}]" | |
| if not isinstance(language, dict): | |
| errors.append(f"{prefix}: entry must be an object") | |
| continue | |
| _validate_language_entry(prefix, language, errors, warnings) | |
| language_id = language.get("id") | |
| if isinstance(language_id, str): | |
| if language_id in ids: | |
| errors.append(f"{prefix}: duplicate id {language_id!r}") | |
| ids.add(language_id) | |
| priority = language.get("priority") | |
| if isinstance(priority, int): | |
| if priority in priorities: | |
| errors.append(f"{prefix}: duplicate priority {priority}") | |
| priorities.add(priority) | |
| if language.get("counts_toward_nepal_language_target") is True: | |
| if isinstance(language_id, str): | |
| counted_ids.append(language_id) | |
| elif language_id == "english": | |
| pass | |
| elif language.get("counts_toward_nepal_language_target") is False: | |
| warnings.append(f"{prefix}: non-English language is not counted toward Nepal target") | |
| if language.get("first_target") is True and isinstance(language_id, str): | |
| first_target_ids.append(language_id) | |
| scripts = language.get("scripts") | |
| if isinstance(language_id, str) and isinstance(scripts, list): | |
| primary_scripts[language_id] = [ | |
| str(script.get("script")) for script in scripts if isinstance(script, dict) and script.get("primary") is True | |
| ] | |
| if not primary_scripts[language_id]: | |
| errors.append(f"{prefix}: at least one script must be primary") | |
| if verify_local_paths: | |
| missing_local_paths.extend(_collect_missing_local_paths(prefix, language)) | |
| if len(counted_ids) < min_nepal_languages: | |
| errors.append(f"registry counts {len(counted_ids)} Nepal languages; minimum is {min_nepal_languages}") | |
| if "english" in counted_ids: | |
| errors.append("english must not count toward the Nepal-language target") | |
| if require_limbu_first: | |
| _validate_limbu_first(languages, errors) | |
| errors.extend(f"missing local path: {path_value}" for path_value in missing_local_paths) | |
| audit = LanguageRegistryAudit( | |
| registry_path=str(path), | |
| registry_id=str(payload.get("registry_id", "")), | |
| passed=not errors, | |
| language_count=len(languages), | |
| counted_nepal_language_count=len(counted_ids), | |
| first_target_ids=first_target_ids, | |
| primary_scripts=primary_scripts, | |
| missing_local_paths=missing_local_paths, | |
| warnings=warnings, | |
| errors=errors, | |
| ) | |
| if out_dir is not None: | |
| out = Path(out_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| json_path = out / "language-registry-audit.json" | |
| md_path = out / "language-registry-audit.md" | |
| json_path.write_text(json.dumps(audit.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| _write_language_registry_markdown(md_path, audit, languages) | |
| audit.summary_json_path = str(json_path) | |
| audit.summary_md_path = str(md_path) | |
| return audit | |
| def audit_font_renderability( | |
| registry_path: str | Path, | |
| out_dir: str | Path | None = None, | |
| *, | |
| primary_only: bool = True, | |
| min_coverage_ratio: float = 0.95, | |
| render_samples: bool = False, | |
| ) -> FontRenderabilityAudit: | |
| path = Path(registry_path) | |
| payload = _load_json_object(path) | |
| languages = payload.get("languages") | |
| if not isinstance(languages, list) or not languages: | |
| raise DataValidationError(f"language registry must contain a non-empty languages list: {path}") | |
| if min_coverage_ratio < 0 or min_coverage_ratio > 1: | |
| raise DataValidationError("min_coverage_ratio must be between 0 and 1") | |
| out = Path(out_dir) if out_dir is not None else None | |
| sample_dir = out / "font-samples" if out is not None else None | |
| if out is not None: | |
| out.mkdir(parents=True, exist_ok=True) | |
| if sample_dir is not None and render_samples: | |
| sample_dir.mkdir(parents=True, exist_ok=True) | |
| pillow_available = _pillow_available() | |
| warnings: list[str] = [] | |
| errors: list[str] = [] | |
| candidates: list[FontCandidateAudit] = [] | |
| script_keys: set[str] = set() | |
| passing_scripts: list[str] = [] | |
| failing_scripts: list[str] = [] | |
| if render_samples and not pillow_available: | |
| warnings.append("Pillow is not installed; sample rendering was requested but skipped") | |
| for language in languages: | |
| if not isinstance(language, dict): | |
| continue | |
| language_id = str(language.get("id", "")) | |
| font_candidates = [item for item in language.get("font_candidates", []) if isinstance(item, dict)] | |
| scripts = [item for item in language.get("scripts", []) if isinstance(item, dict)] | |
| for script in scripts: | |
| if primary_only and script.get("primary") is not True: | |
| continue | |
| script_name = str(script.get("script", "")) | |
| unicode_ranges = script.get("unicode_ranges", []) | |
| if not isinstance(unicode_ranges, list): | |
| unicode_ranges = [] | |
| required = _expand_unicode_ranges(unicode_ranges) | |
| script_key = f"{language_id}:{script_name}" | |
| script_keys.add(script_key) | |
| script_candidates = [ | |
| _audit_font_candidate( | |
| language_id, | |
| script_name, | |
| required, | |
| candidate, | |
| sample_dir=sample_dir, | |
| render_sample=render_samples and pillow_available, | |
| ) | |
| for candidate in font_candidates | |
| ] | |
| candidates.extend(script_candidates) | |
| if any(candidate.coverage_ratio >= min_coverage_ratio for candidate in script_candidates): | |
| passing_scripts.append(script_key) | |
| else: | |
| failing_scripts.append(script_key) | |
| errors.append(f"{script_key}: no font candidate covers at least {min_coverage_ratio:.2%} of required codepoints") | |
| audit = FontRenderabilityAudit( | |
| registry_path=str(path), | |
| passed=not errors, | |
| language_count=len(languages), | |
| script_count=len(script_keys), | |
| candidate_count=len(candidates), | |
| render_requested=render_samples, | |
| pillow_available=pillow_available, | |
| passing_scripts=passing_scripts, | |
| failing_scripts=failing_scripts, | |
| candidates=candidates, | |
| warnings=warnings, | |
| errors=errors, | |
| ) | |
| if out is not None: | |
| json_path = out / "font-renderability-audit.json" | |
| md_path = out / "font-renderability-audit.md" | |
| json_path.write_text(json.dumps(audit.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| _write_font_renderability_markdown(md_path, audit) | |
| audit.summary_json_path = str(json_path) | |
| audit.summary_md_path = str(md_path) | |
| return audit | |
| def _load_json_object(path: Path) -> dict[str, Any]: | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except FileNotFoundError as exc: | |
| raise DataValidationError(f"language registry does not exist: {path}") from exc | |
| except json.JSONDecodeError as exc: | |
| raise DataValidationError(f"language registry is invalid JSON: {path}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise DataValidationError(f"language registry must be a JSON object: {path}") | |
| return payload | |
| def _validate_language_entry(prefix: str, language: dict[str, Any], errors: list[str], warnings: list[str]) -> None: | |
| for field_name in LANGUAGE_REQUIRED_FIELDS: | |
| if field_name not in language: | |
| errors.append(f"{prefix}: missing required field {field_name!r}") | |
| if "iso_639_3" not in language: | |
| warnings.append(f"{prefix}: iso_639_3 missing; set null only after confirming no code exists") | |
| elif language["iso_639_3"] is not None and not _is_non_empty_string(language["iso_639_3"]): | |
| errors.append(f"{prefix}: iso_639_3 must be a non-empty string or null") | |
| if not _is_non_empty_string(language.get("id")): | |
| errors.append(f"{prefix}: id must be a non-empty string") | |
| if not _is_non_empty_string(language.get("name")): | |
| errors.append(f"{prefix}: name must be a non-empty string") | |
| if not isinstance(language.get("priority"), int): | |
| errors.append(f"{prefix}: priority must be an integer") | |
| if not isinstance(language.get("counts_toward_nepal_language_target"), bool): | |
| errors.append(f"{prefix}: counts_toward_nepal_language_target must be boolean") | |
| if not isinstance(language.get("first_target"), bool): | |
| errors.append(f"{prefix}: first_target must be boolean") | |
| if not _is_non_empty_string(language.get("paddle_dictionary")): | |
| errors.append(f"{prefix}: paddle_dictionary must be a non-empty string") | |
| if not isinstance(language.get("gorkhapatra_language_labels"), list): | |
| errors.append(f"{prefix}: gorkhapatra_language_labels must be a list") | |
| _validate_scripts(prefix, language.get("scripts"), errors) | |
| _validate_non_empty_list_of_objects(prefix, "font_candidates", language.get("font_candidates"), errors) | |
| _validate_non_empty_list_of_objects(prefix, "corpora", language.get("corpora"), errors) | |
| _validate_non_empty_list_of_objects(prefix, "validation_sources", language.get("validation_sources"), errors) | |
| _validate_normalization(prefix, language.get("normalization"), errors) | |
| def _validate_scripts(prefix: str, scripts: Any, errors: list[str]) -> None: | |
| if not isinstance(scripts, list) or not scripts: | |
| errors.append(f"{prefix}: scripts must be a non-empty list") | |
| return | |
| for script_index, script in enumerate(scripts): | |
| script_prefix = f"{prefix}.scripts[{script_index}]" | |
| if not isinstance(script, dict): | |
| errors.append(f"{script_prefix}: script entry must be an object") | |
| continue | |
| for field_name in SCRIPT_REQUIRED_FIELDS: | |
| if field_name not in script: | |
| errors.append(f"{script_prefix}: missing required field {field_name!r}") | |
| if not _is_non_empty_string(script.get("script")): | |
| errors.append(f"{script_prefix}: script must be a non-empty string") | |
| if not isinstance(script.get("primary"), bool): | |
| errors.append(f"{script_prefix}: primary must be boolean") | |
| if not _is_non_empty_string(script.get("status")): | |
| errors.append(f"{script_prefix}: status must be a non-empty string") | |
| ranges = script.get("unicode_ranges") | |
| if not isinstance(ranges, list) or not ranges: | |
| errors.append(f"{script_prefix}: unicode_ranges must be a non-empty list") | |
| continue | |
| for range_index, value in enumerate(ranges): | |
| if not isinstance(value, str) or not UNICODE_RANGE_RE.match(value): | |
| errors.append(f"{script_prefix}.unicode_ranges[{range_index}]: invalid Unicode range {value!r}") | |
| def _validate_non_empty_list_of_objects(prefix: str, field_name: str, value: Any, errors: list[str]) -> None: | |
| if not isinstance(value, list) or not value: | |
| errors.append(f"{prefix}: {field_name} must be a non-empty list") | |
| return | |
| for index, item in enumerate(value): | |
| if not isinstance(item, dict): | |
| errors.append(f"{prefix}.{field_name}[{index}]: entry must be an object") | |
| def _validate_normalization(prefix: str, value: Any, errors: list[str]) -> None: | |
| if not isinstance(value, dict): | |
| errors.append(f"{prefix}: normalization must be an object") | |
| return | |
| if value.get("unicode_form") not in {"NFC", "NFKC", "none"}: | |
| errors.append(f"{prefix}: normalization.unicode_form must be NFC, NFKC, or none") | |
| if not _is_non_empty_string(value.get("policy")): | |
| errors.append(f"{prefix}: normalization.policy must be a non-empty string") | |
| def _validate_limbu_first(languages: list[Any], errors: list[str]) -> None: | |
| first = languages[0] if languages else {} | |
| if not isinstance(first, dict): | |
| errors.append("first language entry must be Limbu") | |
| return | |
| if first.get("id") != "limbu": | |
| errors.append("first language entry must have id 'limbu'") | |
| if first.get("priority") != 1: | |
| errors.append("Limbu must have priority 1") | |
| if first.get("first_target") is not True: | |
| errors.append("Limbu must be marked first_target=true") | |
| scripts = first.get("scripts") | |
| if isinstance(scripts, list): | |
| primary_scripts = [script.get("script") for script in scripts if isinstance(script, dict) and script.get("primary") is True] | |
| if "Limbu / Sirijonga" not in primary_scripts: | |
| errors.append("Limbu must have primary script 'Limbu / Sirijonga'") | |
| def _collect_missing_local_paths(prefix: str, language: dict[str, Any]) -> list[str]: | |
| missing: list[str] = [] | |
| for field_name in ("corpora", "validation_sources", "font_candidates"): | |
| values = language.get(field_name) | |
| if not isinstance(values, list): | |
| continue | |
| for index, item in enumerate(values): | |
| if not isinstance(item, dict): | |
| continue | |
| path_value = item.get("path") | |
| status = item.get("status") | |
| if not _should_verify_local_path(path_value, status, item.get("host")): | |
| continue | |
| candidate = Path(path_value) | |
| if not candidate.exists(): | |
| missing.append(f"{prefix}.{field_name}[{index}].path={path_value}") | |
| return missing | |
| def _audit_font_candidate( | |
| language_id: str, | |
| script_name: str, | |
| required: set[int], | |
| candidate: dict[str, Any], | |
| *, | |
| sample_dir: Path | None, | |
| render_sample: bool, | |
| ) -> FontCandidateAudit: | |
| font_name = str(candidate.get("name", "")) | |
| font_path = str(candidate.get("path", "")) | |
| status = str(candidate.get("status", "")) | |
| path = Path(font_path) if font_path and not font_path.startswith(("TBD", "system_or_installable_font")) else None | |
| path_exists = path.exists() if path is not None else False | |
| supported_font_file = path is not None and path.suffix.lower() in {".ttf", ".otf", ".ttc"} | |
| covered: set[int] = set() | |
| error: str | None = None | |
| if path_exists and supported_font_file and path is not None: | |
| try: | |
| covered = _read_font_codepoints(path) | |
| except (OSError, ValueError, struct.error) as exc: | |
| error = str(exc) | |
| elif font_path == "TBD": | |
| error = "font path is TBD" | |
| elif not path_exists: | |
| error = "font path does not exist" | |
| elif not supported_font_file: | |
| error = "unsupported font file type" | |
| covered_required = required.intersection(covered) | |
| missing = sorted(required.difference(covered))[:20] | |
| coverage_ratio = len(covered_required) / len(required) if required else 0.0 | |
| audit = FontCandidateAudit( | |
| language_id=language_id, | |
| script=script_name, | |
| font_name=font_name, | |
| font_path=font_path, | |
| status=status, | |
| path_exists=path_exists, | |
| supported_font_file=bool(supported_font_file), | |
| required_codepoints=len(required), | |
| covered_codepoints=len(covered_required), | |
| coverage_ratio=coverage_ratio, | |
| missing_sample=[_format_codepoint(codepoint) for codepoint in missing], | |
| error=error, | |
| ) | |
| if render_sample: | |
| _render_font_sample(audit, required, path, sample_dir) | |
| return audit | |
| def _expand_unicode_ranges(values: list[Any]) -> set[int]: | |
| codepoints: set[int] = set() | |
| for value in values: | |
| if not isinstance(value, str) or not UNICODE_RANGE_RE.match(value): | |
| continue | |
| if ".." in value: | |
| start_text, end_text = value.split("..", 1) | |
| start = int(start_text.removeprefix("U+"), 16) | |
| end = int(end_text.removeprefix("U+"), 16) | |
| codepoints.update(range(start, end + 1)) | |
| else: | |
| codepoints.add(int(value.removeprefix("U+"), 16)) | |
| return {codepoint for codepoint in codepoints if not unicodedata.category(chr(codepoint)).startswith("C")} | |
| def _read_font_codepoints(path: Path) -> set[int]: | |
| data = path.read_bytes() | |
| offsets = _font_face_offsets(data) | |
| codepoints: set[int] = set() | |
| for offset in offsets: | |
| codepoints.update(_read_face_codepoints(data, offset)) | |
| return codepoints | |
| def _font_face_offsets(data: bytes) -> list[int]: | |
| if len(data) < 12: | |
| raise ValueError("font file is too small") | |
| if data[:4] == b"ttcf": | |
| if len(data) < 12: | |
| raise ValueError("TTC header is truncated") | |
| count = struct.unpack_from(">L", data, 8)[0] | |
| offsets_end = 12 + count * 4 | |
| if len(data) < offsets_end: | |
| raise ValueError("TTC face offset table is truncated") | |
| return [struct.unpack_from(">L", data, 12 + index * 4)[0] for index in range(count)] | |
| return [0] | |
| def _read_face_codepoints(data: bytes, face_offset: int) -> set[int]: | |
| if face_offset + 12 > len(data): | |
| raise ValueError("font face offset is outside file") | |
| table_count = struct.unpack_from(">H", data, face_offset + 4)[0] | |
| table_dir = face_offset + 12 | |
| cmap_offset: int | None = None | |
| cmap_length: int | None = None | |
| for index in range(table_count): | |
| record = table_dir + index * 16 | |
| if record + 16 > len(data): | |
| raise ValueError("font table directory is truncated") | |
| tag = data[record : record + 4] | |
| if tag == b"cmap": | |
| cmap_offset = struct.unpack_from(">L", data, record + 8)[0] | |
| cmap_length = struct.unpack_from(">L", data, record + 12)[0] | |
| break | |
| if cmap_offset is None or cmap_length is None: | |
| return set() | |
| if cmap_offset + cmap_length > len(data): | |
| raise ValueError("cmap table is outside file") | |
| return _read_cmap_codepoints(data, cmap_offset) | |
| def _read_cmap_codepoints(data: bytes, cmap_offset: int) -> set[int]: | |
| if cmap_offset + 4 > len(data): | |
| raise ValueError("cmap header is truncated") | |
| subtable_count = struct.unpack_from(">H", data, cmap_offset + 2)[0] | |
| subtables: list[tuple[int, int, int]] = [] | |
| for index in range(subtable_count): | |
| record = cmap_offset + 4 + index * 8 | |
| if record + 8 > len(data): | |
| raise ValueError("cmap encoding records are truncated") | |
| platform_id, encoding_id, sub_offset = struct.unpack_from(">HHL", data, record) | |
| subtables.append((platform_id, encoding_id, cmap_offset + sub_offset)) | |
| preferred = sorted(subtables, key=lambda item: _cmap_preference(item[0], item[1])) | |
| all_codepoints: set[int] = set() | |
| for _, _, subtable_offset in preferred: | |
| if subtable_offset + 2 > len(data): | |
| continue | |
| fmt = struct.unpack_from(">H", data, subtable_offset)[0] | |
| try: | |
| if fmt == 12: | |
| all_codepoints.update(_read_cmap_format_12(data, subtable_offset)) | |
| elif fmt == 4: | |
| all_codepoints.update(_read_cmap_format_4(data, subtable_offset)) | |
| except (ValueError, struct.error): | |
| continue | |
| if all_codepoints: | |
| return all_codepoints | |
| return all_codepoints | |
| def _cmap_preference(platform_id: int, encoding_id: int) -> int: | |
| if platform_id == 3 and encoding_id == 10: | |
| return 0 | |
| if platform_id == 0: | |
| return 1 | |
| if platform_id == 3 and encoding_id in {1, 0}: | |
| return 2 | |
| return 3 | |
| def _read_cmap_format_12(data: bytes, offset: int) -> set[int]: | |
| if offset + 16 > len(data): | |
| raise ValueError("cmap format 12 header is truncated") | |
| group_count = struct.unpack_from(">L", data, offset + 12)[0] | |
| codepoints: set[int] = set() | |
| for index in range(group_count): | |
| record = offset + 16 + index * 12 | |
| if record + 12 > len(data): | |
| raise ValueError("cmap format 12 groups are truncated") | |
| start, end, _glyph = struct.unpack_from(">LLL", data, record) | |
| if start <= end: | |
| codepoints.update(range(start, end + 1)) | |
| return codepoints | |
| def _read_cmap_format_4(data: bytes, offset: int) -> set[int]: | |
| if offset + 14 > len(data): | |
| raise ValueError("cmap format 4 header is truncated") | |
| length = struct.unpack_from(">H", data, offset + 2)[0] | |
| if offset + length > len(data): | |
| raise ValueError("cmap format 4 table is truncated") | |
| seg_count_x2 = struct.unpack_from(">H", data, offset + 6)[0] | |
| seg_count = seg_count_x2 // 2 | |
| end_codes_offset = offset + 14 | |
| start_codes_offset = end_codes_offset + seg_count * 2 + 2 | |
| id_delta_offset = start_codes_offset + seg_count * 2 | |
| id_range_offset_offset = id_delta_offset + seg_count * 2 | |
| glyph_array_offset = id_range_offset_offset + seg_count * 2 | |
| if glyph_array_offset > offset + length: | |
| raise ValueError("cmap format 4 arrays are truncated") | |
| codepoints: set[int] = set() | |
| for index in range(seg_count): | |
| end_code = struct.unpack_from(">H", data, end_codes_offset + index * 2)[0] | |
| start_code = struct.unpack_from(">H", data, start_codes_offset + index * 2)[0] | |
| id_delta = struct.unpack_from(">h", data, id_delta_offset + index * 2)[0] | |
| id_range_offset = struct.unpack_from(">H", data, id_range_offset_offset + index * 2)[0] | |
| if start_code == 0xFFFF and end_code == 0xFFFF: | |
| continue | |
| for codepoint in range(start_code, end_code + 1): | |
| if id_range_offset == 0: | |
| glyph_id = (codepoint + id_delta) & 0xFFFF | |
| else: | |
| glyph_offset = id_range_offset_offset + index * 2 + id_range_offset + (codepoint - start_code) * 2 | |
| if glyph_offset + 2 > offset + length: | |
| glyph_id = 0 | |
| else: | |
| glyph_id = struct.unpack_from(">H", data, glyph_offset)[0] | |
| if glyph_id: | |
| glyph_id = (glyph_id + id_delta) & 0xFFFF | |
| if glyph_id: | |
| codepoints.add(codepoint) | |
| return codepoints | |
| def _render_font_sample(audit: FontCandidateAudit, required: set[int], path: Path | None, sample_dir: Path | None) -> None: | |
| if path is None or sample_dir is None or not path.exists(): | |
| audit.render_status = "skipped_missing_font" | |
| return | |
| try: | |
| from PIL import Image, ImageDraw, ImageFont | |
| except ImportError: | |
| audit.render_status = "skipped_pillow_missing" | |
| return | |
| sample_text = _sample_text(required) | |
| if not sample_text: | |
| audit.render_status = "skipped_empty_sample" | |
| return | |
| try: | |
| font = ImageFont.truetype(str(path), 48) | |
| image = Image.new("RGB", (1200, 160), "white") | |
| draw = ImageDraw.Draw(image) | |
| draw.text((24, 42), sample_text, fill="black", font=font) | |
| filename = f"{_slug(audit.language_id)}-{_slug(audit.script)}-{_slug(audit.font_name)}.png" | |
| output = sample_dir / filename | |
| image.save(output) | |
| audit.render_sample_path = str(output) | |
| audit.render_status = "rendered" | |
| except (OSError, ValueError) as exc: | |
| audit.render_status = "failed" | |
| audit.error = f"{audit.error}; render failed: {exc}" if audit.error else f"render failed: {exc}" | |
| def _sample_text(required: set[int]) -> str: | |
| chars = [chr(codepoint) for codepoint in sorted(required) if unicodedata.category(chr(codepoint))[0] in {"L", "M", "N"}] | |
| return "".join(chars[:24]) | |
| def _pillow_available() -> bool: | |
| try: | |
| import PIL # noqa: F401 | |
| except ImportError: | |
| return False | |
| return True | |
| def _format_codepoint(codepoint: int) -> str: | |
| return f"U+{codepoint:04X}" | |
| def _slug(value: str) -> str: | |
| slug = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-").lower() | |
| return slug or "sample" | |
| def _should_verify_local_path(path_value: Any, status: Any, host: Any = None) -> bool: | |
| if not isinstance(path_value, str) or not path_value: | |
| return False | |
| if isinstance(host, str) and host and host != getpass.getuser(): | |
| return False | |
| if status not in {"exists_local", "local_verified_directory", "schema_audit_pending"}: | |
| return False | |
| if path_value.startswith(("TBD", "huggingface:", "system_or_installable_font")): | |
| return False | |
| return path_value.startswith("/") | |
| def _write_language_registry_markdown(path: Path, audit: LanguageRegistryAudit, languages: list[Any]) -> None: | |
| lines = [ | |
| "# Language Registry Audit", | |
| "", | |
| f"- Registry: `{audit.registry_id}`", | |
| f"- Passed: `{audit.passed}`", | |
| f"- Languages: `{audit.language_count}`", | |
| f"- Nepal-language targets counted: `{audit.counted_nepal_language_count}`", | |
| f"- First targets: `{', '.join(audit.first_target_ids) or 'none'}`", | |
| "", | |
| "## Languages", | |
| "", | |
| "| Priority | ID | Name | Counted | Primary scripts | Validation labels |", | |
| "| ---: | --- | --- | --- | --- | --- |", | |
| ] | |
| sorted_languages = sorted( | |
| [language for language in languages if isinstance(language, dict)], | |
| key=lambda language: language.get("priority") if isinstance(language.get("priority"), int) else 9999, | |
| ) | |
| for language in sorted_languages: | |
| language_id = str(language.get("id", "")) | |
| primary_scripts = ", ".join(audit.primary_scripts.get(language_id, [])) | |
| labels = ", ".join(str(label) for label in language.get("gorkhapatra_language_labels", [])) | |
| lines.append( | |
| "| " | |
| f"{language.get('priority', '')} | " | |
| f"`{language_id}` | " | |
| f"{language.get('name', '')} | " | |
| f"`{language.get('counts_toward_nepal_language_target', '')}` | " | |
| f"{primary_scripts} | " | |
| f"{labels} |" | |
| ) | |
| if audit.errors: | |
| lines.extend(["", "## Errors", ""]) | |
| lines.extend(f"- {error}" for error in audit.errors) | |
| if audit.warnings: | |
| lines.extend(["", "## Warnings", ""]) | |
| lines.extend(f"- {warning}" for warning in audit.warnings) | |
| if audit.missing_local_paths: | |
| lines.extend(["", "## Missing Local Paths", ""]) | |
| lines.extend(f"- `{missing}`" for missing in audit.missing_local_paths) | |
| path.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| def _write_font_renderability_markdown(path: Path, audit: FontRenderabilityAudit) -> None: | |
| lines = [ | |
| "# Font Renderability Audit", | |
| "", | |
| f"- Passed: `{audit.passed}`", | |
| f"- Scripts checked: `{audit.script_count}`", | |
| f"- Font candidates checked: `{audit.candidate_count}`", | |
| f"- Render requested: `{audit.render_requested}`", | |
| f"- Pillow available: `{audit.pillow_available}`", | |
| "", | |
| "## Candidate Coverage", | |
| "", | |
| "| Language | Script | Font | Exists | Coverage | Render | Error |", | |
| "| --- | --- | --- | --- | ---: | --- | --- |", | |
| ] | |
| for candidate in audit.candidates: | |
| error = candidate.error or "" | |
| lines.append( | |
| "| " | |
| f"`{candidate.language_id}` | " | |
| f"{candidate.script} | " | |
| f"{candidate.font_name} | " | |
| f"`{candidate.path_exists}` | " | |
| f"{candidate.coverage_ratio:.1%} | " | |
| f"`{candidate.render_status}` | " | |
| f"{error} |" | |
| ) | |
| if audit.errors: | |
| lines.extend(["", "## Errors", ""]) | |
| lines.extend(f"- {error}" for error in audit.errors) | |
| if audit.warnings: | |
| lines.extend(["", "## Warnings", ""]) | |
| lines.extend(f"- {warning}" for warning in audit.warnings) | |
| path.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| def _is_non_empty_string(value: Any) -> bool: | |
| return isinstance(value, str) and bool(value.strip()) | |