Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/src/ocrtech/lepcha_legacy.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 18.7 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/lepcha_legacy.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/src/ocrtech/lepcha_legacy.py
-
curl -L -o lepcha_legacy.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/lepcha_legacy.py
18.7 kB
| """Convert JG Lepcha legacy-encoded text to Unicode Lepcha (Róng script). | |
| The Sikkim Herald Lepcha edition is typeset in *Jason Glavy's* ``JG Lepcha`` | |
| custom-encoded TrueType font: a font that stuffs the Lepcha (Róng) glyph | |
| repertoire into a Times-New-Roman shell, addressing each glyph by a single | |
| legacy byte (Latin code point). SIL publishes a TECkit mapping | |
| (``silnrsi/wsresources`` -> ``scripts/Lepc/legacy/jg-lepcha/mappings/ | |
| JGLepcha.map``) that converts that legacy byte stream to the Lepcha Unicode | |
| block (U+1C00-U+1C4F). | |
| The map is a *two-pass* TECkit conversion: | |
| ``pass(Byte_Unicode)`` | |
| ``ByteClass``/``UniClass`` declarations plus explicit single-byte and | |
| composite (multi-codepoint) rules map the legacy bytes to Lepcha codepoints | |
| in the order the bytes appear in the font's *visual* layout. This pass also | |
| contains the one context rule ``0x61 / ^[DepVow] _ > U+1C26`` (a bare ``a`` | |
| after a dependent vowel is the independent vowel A, not the dep. vowel AA). | |
| ``pass(Unicode)`` | |
| A reordering pass that rewrites runs of ``C (.) (R) (Y) (V) (F) (^)`` into | |
| the canonical Lepcha storage order. The legacy font keys some dependent | |
| vowels and final-consonant signs *before* their base consonant (visual | |
| order); this pass moves the base consonant to the front so the output is in | |
| logical/Unicode order, not visual order. | |
| This module parses the TECkit map natively (no ``teckit_compile`` dependency, | |
| which is unavailable on this machine) and applies both passes. It mirrors the | |
| structure of :mod:`ocrtech.kiratrai_legacy`. | |
| NOTE on the Sikkim Herald source: in the *current* Sikkim Herald Lepcha PDFs the | |
| Lepcha body text is rasterised into image strips, so there is no recoverable | |
| legacy byte stream for the body columns (only the English masthead/colophon and | |
| inter-word punctuation are live text). This converter is therefore validated | |
| against the SIL map's own round-trip rules and a synthetic JG Lepcha byte | |
| sample; it will convert any genuine JG-Lepcha-encoded byte stream should one | |
| become available (e.g. an older live-text edition or the source DTP files). | |
| """ | |
| from __future__ import annotations | |
| import re | |
| import unicodedata | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any | |
| from .errors import DataValidationError | |
| DEFAULT_LEPCHA_LEGACY_MAP = Path("data/mappings/lepcha-legacy/JGLepcha.map") | |
| LEPCHA_LO, LEPCHA_HI = 0x1C00, 0x1C4F | |
| _LEPCHA_CODEPOINT_RE = re.compile(r"[ᰀ-ᱏ]") | |
| _BYTE_TOKEN_RE = re.compile(r"0x([0-9A-Fa-f]{2})") | |
| _UNI_TOKEN_RE = re.compile(r"U\+([0-9A-Fa-f]{4,6})") | |
| _BYTECLASS_RE = re.compile(r"^ByteClass\s*\[([^\]]+)\]\s*=\s*\((.*)\)\s*$") | |
| _UNICLASS_RE = re.compile(r"^UniClass\s*\[([^\]]+)\]\s*=\s*\((.*)\)\s*$") | |
| # Pass(Unicode) classes are declared as bare ``Class[name] = (...)``. | |
| _PLAINCLASS_RE = re.compile(r"^Class\s*\[([^\]]+)\]\s*=\s*\((.*)\)\s*$") | |
| # A simple ``[byteclass] <> [uniclass]`` Pass-1 rule. | |
| _CLASS_RULE_RE = re.compile(r"^\[([^\]]+)\]\s*<?>\s*\[([^\]]+)\]\s*$") | |
| # A Pass-2 reorder rule: ``[Cls]=v [Cls]=c ... <> @c @v ...`` | |
| _BOUND_CLASS_RE = re.compile(r"\[([^\]]+)\]\s*=\s*([A-Za-z]\w*)") | |
| _VAR_REF_RE = re.compile(r"@([A-Za-z]\w*)") | |
| # The single context rule: ``0x61 / ^[DepVow] _ > U+1C26`` | |
| _CONTEXT_RULE_RE = re.compile( | |
| r"^0x([0-9A-Fa-f]{2})\s*/\s*\^\s*\[([^\]]+)\]\s*_\s*>\s*U\+([0-9A-Fa-f]{4,6})\s*$" | |
| ) | |
| def _split_tokens(body: str) -> list[str]: | |
| # The JG Lepcha map writes ranges both spaced (``0x00 .. 0x1F``) and compact | |
| # (``U+1C00..U+1C23``). Normalise the compact form to the spaced form so the | |
| # range handling below sees a uniform ``A .. B`` token stream. | |
| body = re.sub(r"\s*\.\.\s*", " .. ", body) | |
| return [t for t in re.split(r"\s+", body.strip()) if t] | |
| def _expand_byte_tokens(body: str) -> tuple[int, ...]: | |
| """Expand a TECkit byte-class body into an ordered tuple of byte values. | |
| Supports explicit ``0xNN`` tokens and ``0xNN .. 0xMM`` ranges. | |
| """ | |
| values: list[int] = [] | |
| tokens = _split_tokens(body) | |
| index = 0 | |
| while index < len(tokens): | |
| token = tokens[index] | |
| if index + 2 < len(tokens) and tokens[index + 1] == "..": | |
| start = int(token, 16) | |
| end = int(tokens[index + 2], 16) | |
| if end < start: | |
| raise DataValidationError( | |
| f"invalid byte range in map: {token}..{tokens[index + 2]}" | |
| ) | |
| values.extend(range(start, end + 1)) | |
| index += 3 | |
| continue | |
| if not _BYTE_TOKEN_RE.fullmatch(token): | |
| raise DataValidationError(f"unparseable byte token in map: {token!r}") | |
| values.append(int(token, 16)) | |
| index += 1 | |
| return tuple(values) | |
| def _expand_uni_tokens(body: str) -> tuple[int, ...]: | |
| """Expand a TECkit Unicode-class body into an ordered tuple of codepoints.""" | |
| values: list[int] = [] | |
| tokens = _split_tokens(body) | |
| index = 0 | |
| while index < len(tokens): | |
| token = tokens[index] | |
| if index + 2 < len(tokens) and tokens[index + 1] == "..": | |
| start_match = _UNI_TOKEN_RE.fullmatch(token) | |
| end_match = _UNI_TOKEN_RE.fullmatch(tokens[index + 2]) | |
| if start_match is None or end_match is None: | |
| raise DataValidationError( | |
| f"invalid unicode range in map: {token}..{tokens[index + 2]}" | |
| ) | |
| start = int(start_match.group(1), 16) | |
| end = int(end_match.group(1), 16) | |
| if end < start: | |
| raise DataValidationError( | |
| f"invalid unicode range in map: {token}..{tokens[index + 2]}" | |
| ) | |
| values.extend(range(start, end + 1)) | |
| index += 3 | |
| continue | |
| match = _UNI_TOKEN_RE.fullmatch(token) | |
| if match is None: | |
| raise DataValidationError(f"unparseable unicode token in map: {token!r}") | |
| values.append(int(match.group(1), 16)) | |
| index += 1 | |
| return tuple(values) | |
| class _ReorderRule: | |
| """A Pass(Unicode) contextual reorder rule. | |
| ``slots`` is an ordered list of ``(class_name, var_name)`` describing the LHS | |
| match sequence; ``output_vars`` is the ordered list of variable names on the | |
| RHS. Each slot matches exactly one input codepoint that belongs to the named | |
| class; the rule rewrites the matched run by emitting the bound codepoints in | |
| the RHS order. | |
| """ | |
| slots: tuple[tuple[str, str], ...] | |
| output_vars: tuple[str, ...] | |
| class LepchaLegacyConversion: | |
| legacy_text: str | |
| unicode_text: str | |
| lepcha_char_count: int | |
| replacement_count: int | |
| unmapped_codepoints: list[str] | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "legacy_text": self.legacy_text, | |
| "unicode_text": self.unicode_text, | |
| "lepcha_char_count": self.lepcha_char_count, | |
| "replacement_count": self.replacement_count, | |
| "unmapped_codepoints": self.unmapped_codepoints, | |
| } | |
| class LepchaLegacyConverter: | |
| """Native two-pass reader/applier for SIL's ``JGLepcha.map`` TECkit mapping. | |
| Pass 1 (``Byte_Unicode``) flattens every byte-class and explicit rule into | |
| ``(source_bytes, target_codepoints)`` pairs, applied longest-match-first so | |
| composite/conjunct multi-byte rules win over single-byte class rules. The one | |
| context rule (bare ``a`` after a dependent vowel -> independent A) is applied | |
| as a guarded special case. | |
| Pass 2 (``Unicode``) applies the contextual reorder rules, longest-LHS-first, | |
| to rewrite visual-order runs into canonical Lepcha storage order. | |
| """ | |
| def __init__( | |
| self, | |
| byte_rules: list[tuple[tuple[int, ...], tuple[int, ...]]], | |
| reorder_rules: list[_ReorderRule], | |
| uni_classes: dict[str, frozenset[int]], | |
| context_rule: tuple[int, frozenset[int], int] | None, | |
| ) -> None: | |
| if not byte_rules: | |
| raise DataValidationError( | |
| "Lepcha legacy converter requires at least one Byte_Unicode rule" | |
| ) | |
| seen: set[tuple[int, ...]] = set() | |
| ordered: list[tuple[tuple[int, ...], tuple[int, ...]]] = [] | |
| for source, target in sorted(byte_rules, key=lambda item: len(item[0]), reverse=True): | |
| if source in seen: | |
| continue | |
| seen.add(source) | |
| ordered.append((source, target)) | |
| self._byte_rules = ordered | |
| # Longest LHS first so the most specific reorder pattern matches. | |
| self._reorder_rules = sorted( | |
| reorder_rules, key=lambda r: len(r.slots), reverse=True | |
| ) | |
| self._uni_classes = uni_classes | |
| self._context_rule = context_rule | |
| def from_map_file( | |
| cls, path: str | Path = DEFAULT_LEPCHA_LEGACY_MAP | |
| ) -> "LepchaLegacyConverter": | |
| map_path = Path(path) | |
| if not map_path.is_file(): | |
| raise DataValidationError(f"Lepcha legacy map does not exist: {map_path}") | |
| # Join backslash line continuations, then strip comments per line. | |
| raw = map_path.read_text(encoding="utf-8-sig") | |
| raw = re.sub(r"\\\s*\n", " ", raw) | |
| byte_classes: dict[str, tuple[int, ...]] = {} | |
| uni_classes_ordered: dict[str, tuple[int, ...]] = {} | |
| plain_classes: dict[str, frozenset[int]] = {} | |
| byte_rules: list[tuple[tuple[int, ...], tuple[int, ...]]] = [] | |
| reorder_rules: list[_ReorderRule] = [] | |
| context_rule: tuple[int, frozenset[int], int] | None = None | |
| current_pass = "" | |
| byte_rule_lines: list[str] = [] | |
| for raw_line in raw.splitlines(): | |
| line = raw_line.split(";", 1)[0].strip() | |
| if not line: | |
| continue | |
| lowered = line.lower() | |
| if lowered.startswith("pass("): | |
| current_pass = lowered | |
| continue | |
| if current_pass == "pass(byte_unicode)": | |
| byte_match = _BYTECLASS_RE.match(line) | |
| if byte_match: | |
| byte_classes[byte_match.group(1).strip()] = _expand_byte_tokens( | |
| byte_match.group(2) | |
| ) | |
| continue | |
| uni_match = _UNICLASS_RE.match(line) | |
| if uni_match: | |
| uni_classes_ordered[uni_match.group(1).strip()] = _expand_uni_tokens( | |
| uni_match.group(2) | |
| ) | |
| continue | |
| byte_rule_lines.append(line) | |
| continue | |
| if current_pass == "pass(unicode)": | |
| plain_match = _PLAINCLASS_RE.match(line) | |
| if plain_match: | |
| plain_classes[plain_match.group(1).strip()] = frozenset( | |
| _expand_uni_tokens(plain_match.group(2)) | |
| ) | |
| continue | |
| rule = cls._parse_reorder_rule(line) | |
| if rule is not None: | |
| reorder_rules.append(rule) | |
| continue | |
| # Second pass over Byte_Unicode rule lines so class rules can reference | |
| # classes regardless of declaration order. | |
| for line in byte_rule_lines: | |
| ctx = _CONTEXT_RULE_RE.match(line) | |
| if ctx: | |
| cls_name = ctx.group(2).strip() | |
| cls_bytes = byte_classes.get(cls_name) | |
| if cls_bytes is None: | |
| raise DataValidationError( | |
| f"context rule references unknown byte class: {cls_name!r}" | |
| ) | |
| # The context class names a *byte* class but the run-time check is | |
| # against already-converted Unicode codepoints. Map the byte class | |
| # to its Unicode equivalents via the paired UniClass. | |
| cls_codepoints = uni_classes_ordered.get(cls_name) | |
| if cls_codepoints is None: | |
| raise DataValidationError( | |
| f"context rule class {cls_name!r} has no UniClass" | |
| ) | |
| context_rule = ( | |
| int(ctx.group(1), 16), | |
| frozenset(cls_codepoints), | |
| int(ctx.group(3), 16), | |
| ) | |
| continue | |
| class_rule = _CLASS_RULE_RE.match(line) | |
| if class_rule: | |
| left_name = class_rule.group(1).strip() | |
| right_name = class_rule.group(2).strip() | |
| left = byte_classes.get(left_name) | |
| right = uni_classes_ordered.get(right_name) | |
| if left is None: | |
| raise DataValidationError( | |
| f"class rule references unknown byte class: {left_name!r}" | |
| ) | |
| if right is None: | |
| raise DataValidationError( | |
| f"class rule references unknown unicode class: {right_name!r}" | |
| ) | |
| if len(left) != len(right): | |
| raise DataValidationError( | |
| f"class rule length mismatch for [{left_name}]>[{right_name}]: " | |
| f"{len(left)} bytes vs {len(right)} codepoints" | |
| ) | |
| for byte_value, codepoint in zip(left, right): | |
| byte_rules.append(((byte_value,), (codepoint,))) | |
| continue | |
| if "<>" in line or ">" in line: | |
| left_text, right_text = ( | |
| line.split("<>", 1) if "<>" in line else line.split(">", 1) | |
| ) | |
| source = _BYTE_TOKEN_RE.findall(left_text) | |
| target = _UNI_TOKEN_RE.findall(right_text) | |
| if source and target: | |
| byte_rules.append( | |
| ( | |
| tuple(int(value, 16) for value in source), | |
| tuple(int(value, 16) for value in target), | |
| ) | |
| ) | |
| return cls(byte_rules, reorder_rules, plain_classes, context_rule) | |
| def _parse_reorder_rule(line: str) -> _ReorderRule | None: | |
| if "<>" not in line: | |
| return None | |
| left, right = line.split("<>", 1) | |
| slots = tuple( | |
| (cls.strip(), var.strip()) | |
| for cls, var in _BOUND_CLASS_RE.findall(left) | |
| ) | |
| output_vars = tuple(_VAR_REF_RE.findall(right)) | |
| if not slots or not output_vars: | |
| return None | |
| # Only keep well-formed permutation rules where every output var is bound. | |
| bound = {var for _, var in slots} | |
| if any(var not in bound for var in output_vars): | |
| return None | |
| return _ReorderRule(slots=slots, output_vars=output_vars) | |
| # ----- Pass 1: Byte_Unicode ------------------------------------------------- | |
| def _apply_byte_pass(self, text: str) -> tuple[str, int, list[str]]: | |
| output: list[str] = [] | |
| unmapped: list[str] = [] | |
| replacements = 0 | |
| index = 0 | |
| length = len(text) | |
| while index < length: | |
| # Context rule: bare 0x61 after a (previously emitted) dependent vowel | |
| # becomes the independent vowel A (U+1C26) rather than dep. vowel AA. | |
| if self._context_rule is not None: | |
| trigger_byte, prev_classes, replacement = self._context_rule | |
| if ( | |
| ord(text[index]) == trigger_byte | |
| and output | |
| and ord(output[-1]) in prev_classes | |
| ): | |
| output.append(chr(replacement)) | |
| replacements += 1 | |
| index += 1 | |
| continue | |
| matched = False | |
| for source, target in self._byte_rules: | |
| if self._matches(text, index, source): | |
| output.extend(chr(value) for value in target) | |
| replacements += 1 | |
| index += len(source) | |
| matched = True | |
| break | |
| if matched: | |
| continue | |
| char = text[index] | |
| output.append(char) | |
| code = ord(char) | |
| if code > 0x7F and not (LEPCHA_LO <= code <= LEPCHA_HI): | |
| unmapped.append(f"U+{code:04X}") | |
| index += 1 | |
| return "".join(output), replacements, unmapped | |
| # ----- Pass 2: Unicode reorder --------------------------------------------- | |
| def _apply_reorder_pass(self, text: str) -> str: | |
| output: list[str] = [] | |
| index = 0 | |
| length = len(text) | |
| while index < length: | |
| applied = False | |
| for rule in self._reorder_rules: | |
| bound = self._match_reorder(text, index, rule) | |
| if bound is not None: | |
| output.extend(bound[var] for var in rule.output_vars) | |
| index += len(rule.slots) | |
| applied = True | |
| break | |
| if applied: | |
| continue | |
| output.append(text[index]) | |
| index += 1 | |
| return "".join(output) | |
| def _match_reorder( | |
| self, text: str, index: int, rule: _ReorderRule | |
| ) -> dict[str, str] | None: | |
| if index + len(rule.slots) > len(text): | |
| return None | |
| bound: dict[str, str] = {} | |
| for offset, (cls_name, var_name) in enumerate(rule.slots): | |
| members = self._uni_classes.get(cls_name) | |
| if members is None: | |
| return None | |
| ch = text[index + offset] | |
| if ord(ch) not in members: | |
| return None | |
| bound[var_name] = ch | |
| return bound | |
| def _matches(text: str, index: int, source: tuple[int, ...]) -> bool: | |
| if index + len(source) > len(text): | |
| return False | |
| return all(ord(text[index + offset]) == code for offset, code in enumerate(source)) | |
| # ----- public -------------------------------------------------------------- | |
| def convert(self, text: str) -> LepchaLegacyConversion: | |
| stage1, replacements, unmapped = self._apply_byte_pass(text) | |
| stage2 = self._apply_reorder_pass(stage1) | |
| converted = unicodedata.normalize("NFC", stage2) | |
| return LepchaLegacyConversion( | |
| legacy_text=text, | |
| unicode_text=converted, | |
| lepcha_char_count=len(_LEPCHA_CODEPOINT_RE.findall(converted)), | |
| replacement_count=replacements, | |
| unmapped_codepoints=sorted(set(unmapped)), | |
| ) | |
| def convert_lepcha_legacy_text( | |
| text: str, map_path: str | Path = DEFAULT_LEPCHA_LEGACY_MAP | |
| ) -> LepchaLegacyConversion: | |
| return LepchaLegacyConverter.from_map_file(map_path).convert(text) | |