prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
18.7 kB
"""Convert JG Lepcha legacy-encoded text to Unicode Lepcha (Róng script).
The Sikkim Herald Lepcha edition is typeset in *Jason Glavy's* ``JG Lepcha``
custom-encoded TrueType font: a font that stuffs the Lepcha (Róng) glyph
repertoire into a Times-New-Roman shell, addressing each glyph by a single
legacy byte (Latin code point). SIL publishes a TECkit mapping
(``silnrsi/wsresources`` -> ``scripts/Lepc/legacy/jg-lepcha/mappings/
JGLepcha.map``) that converts that legacy byte stream to the Lepcha Unicode
block (U+1C00-U+1C4F).
The map is a *two-pass* TECkit conversion:
``pass(Byte_Unicode)``
``ByteClass``/``UniClass`` declarations plus explicit single-byte and
composite (multi-codepoint) rules map the legacy bytes to Lepcha codepoints
in the order the bytes appear in the font's *visual* layout. This pass also
contains the one context rule ``0x61 / ^[DepVow] _ > U+1C26`` (a bare ``a``
after a dependent vowel is the independent vowel A, not the dep. vowel AA).
``pass(Unicode)``
A reordering pass that rewrites runs of ``C (.) (R) (Y) (V) (F) (^)`` into
the canonical Lepcha storage order. The legacy font keys some dependent
vowels and final-consonant signs *before* their base consonant (visual
order); this pass moves the base consonant to the front so the output is in
logical/Unicode order, not visual order.
This module parses the TECkit map natively (no ``teckit_compile`` dependency,
which is unavailable on this machine) and applies both passes. It mirrors the
structure of :mod:`ocrtech.kiratrai_legacy`.
NOTE on the Sikkim Herald source: in the *current* Sikkim Herald Lepcha PDFs the
Lepcha body text is rasterised into image strips, so there is no recoverable
legacy byte stream for the body columns (only the English masthead/colophon and
inter-word punctuation are live text). This converter is therefore validated
against the SIL map's own round-trip rules and a synthetic JG Lepcha byte
sample; it will convert any genuine JG-Lepcha-encoded byte stream should one
become available (e.g. an older live-text edition or the source DTP files).
"""
from __future__ import annotations
import re
import unicodedata
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .errors import DataValidationError
DEFAULT_LEPCHA_LEGACY_MAP = Path("data/mappings/lepcha-legacy/JGLepcha.map")
LEPCHA_LO, LEPCHA_HI = 0x1C00, 0x1C4F
_LEPCHA_CODEPOINT_RE = re.compile(r"[ᰀ-ᱏ]")
_BYTE_TOKEN_RE = re.compile(r"0x([0-9A-Fa-f]{2})")
_UNI_TOKEN_RE = re.compile(r"U\+([0-9A-Fa-f]{4,6})")
_BYTECLASS_RE = re.compile(r"^ByteClass\s*\[([^\]]+)\]\s*=\s*\((.*)\)\s*$")
_UNICLASS_RE = re.compile(r"^UniClass\s*\[([^\]]+)\]\s*=\s*\((.*)\)\s*$")
# Pass(Unicode) classes are declared as bare ``Class[name] = (...)``.
_PLAINCLASS_RE = re.compile(r"^Class\s*\[([^\]]+)\]\s*=\s*\((.*)\)\s*$")
# A simple ``[byteclass] <> [uniclass]`` Pass-1 rule.
_CLASS_RULE_RE = re.compile(r"^\[([^\]]+)\]\s*<?>\s*\[([^\]]+)\]\s*$")
# A Pass-2 reorder rule: ``[Cls]=v [Cls]=c ... <> @c @v ...``
_BOUND_CLASS_RE = re.compile(r"\[([^\]]+)\]\s*=\s*([A-Za-z]\w*)")
_VAR_REF_RE = re.compile(r"@([A-Za-z]\w*)")
# The single context rule: ``0x61 / ^[DepVow] _ > U+1C26``
_CONTEXT_RULE_RE = re.compile(
r"^0x([0-9A-Fa-f]{2})\s*/\s*\^\s*\[([^\]]+)\]\s*_\s*>\s*U\+([0-9A-Fa-f]{4,6})\s*$"
)
def _split_tokens(body: str) -> list[str]:
# The JG Lepcha map writes ranges both spaced (``0x00 .. 0x1F``) and compact
# (``U+1C00..U+1C23``). Normalise the compact form to the spaced form so the
# range handling below sees a uniform ``A .. B`` token stream.
body = re.sub(r"\s*\.\.\s*", " .. ", body)
return [t for t in re.split(r"\s+", body.strip()) if t]
def _expand_byte_tokens(body: str) -> tuple[int, ...]:
"""Expand a TECkit byte-class body into an ordered tuple of byte values.
Supports explicit ``0xNN`` tokens and ``0xNN .. 0xMM`` ranges.
"""
values: list[int] = []
tokens = _split_tokens(body)
index = 0
while index < len(tokens):
token = tokens[index]
if index + 2 < len(tokens) and tokens[index + 1] == "..":
start = int(token, 16)
end = int(tokens[index + 2], 16)
if end < start:
raise DataValidationError(
f"invalid byte range in map: {token}..{tokens[index + 2]}"
)
values.extend(range(start, end + 1))
index += 3
continue
if not _BYTE_TOKEN_RE.fullmatch(token):
raise DataValidationError(f"unparseable byte token in map: {token!r}")
values.append(int(token, 16))
index += 1
return tuple(values)
def _expand_uni_tokens(body: str) -> tuple[int, ...]:
"""Expand a TECkit Unicode-class body into an ordered tuple of codepoints."""
values: list[int] = []
tokens = _split_tokens(body)
index = 0
while index < len(tokens):
token = tokens[index]
if index + 2 < len(tokens) and tokens[index + 1] == "..":
start_match = _UNI_TOKEN_RE.fullmatch(token)
end_match = _UNI_TOKEN_RE.fullmatch(tokens[index + 2])
if start_match is None or end_match is None:
raise DataValidationError(
f"invalid unicode range in map: {token}..{tokens[index + 2]}"
)
start = int(start_match.group(1), 16)
end = int(end_match.group(1), 16)
if end < start:
raise DataValidationError(
f"invalid unicode range in map: {token}..{tokens[index + 2]}"
)
values.extend(range(start, end + 1))
index += 3
continue
match = _UNI_TOKEN_RE.fullmatch(token)
if match is None:
raise DataValidationError(f"unparseable unicode token in map: {token!r}")
values.append(int(match.group(1), 16))
index += 1
return tuple(values)
@dataclass(frozen=True, slots=True)
class _ReorderRule:
"""A Pass(Unicode) contextual reorder rule.
``slots`` is an ordered list of ``(class_name, var_name)`` describing the LHS
match sequence; ``output_vars`` is the ordered list of variable names on the
RHS. Each slot matches exactly one input codepoint that belongs to the named
class; the rule rewrites the matched run by emitting the bound codepoints in
the RHS order.
"""
slots: tuple[tuple[str, str], ...]
output_vars: tuple[str, ...]
@dataclass(frozen=True, slots=True)
class LepchaLegacyConversion:
legacy_text: str
unicode_text: str
lepcha_char_count: int
replacement_count: int
unmapped_codepoints: list[str]
def to_dict(self) -> dict[str, Any]:
return {
"legacy_text": self.legacy_text,
"unicode_text": self.unicode_text,
"lepcha_char_count": self.lepcha_char_count,
"replacement_count": self.replacement_count,
"unmapped_codepoints": self.unmapped_codepoints,
}
class LepchaLegacyConverter:
"""Native two-pass reader/applier for SIL's ``JGLepcha.map`` TECkit mapping.
Pass 1 (``Byte_Unicode``) flattens every byte-class and explicit rule into
``(source_bytes, target_codepoints)`` pairs, applied longest-match-first so
composite/conjunct multi-byte rules win over single-byte class rules. The one
context rule (bare ``a`` after a dependent vowel -> independent A) is applied
as a guarded special case.
Pass 2 (``Unicode``) applies the contextual reorder rules, longest-LHS-first,
to rewrite visual-order runs into canonical Lepcha storage order.
"""
def __init__(
self,
byte_rules: list[tuple[tuple[int, ...], tuple[int, ...]]],
reorder_rules: list[_ReorderRule],
uni_classes: dict[str, frozenset[int]],
context_rule: tuple[int, frozenset[int], int] | None,
) -> None:
if not byte_rules:
raise DataValidationError(
"Lepcha legacy converter requires at least one Byte_Unicode rule"
)
seen: set[tuple[int, ...]] = set()
ordered: list[tuple[tuple[int, ...], tuple[int, ...]]] = []
for source, target in sorted(byte_rules, key=lambda item: len(item[0]), reverse=True):
if source in seen:
continue
seen.add(source)
ordered.append((source, target))
self._byte_rules = ordered
# Longest LHS first so the most specific reorder pattern matches.
self._reorder_rules = sorted(
reorder_rules, key=lambda r: len(r.slots), reverse=True
)
self._uni_classes = uni_classes
self._context_rule = context_rule
@classmethod
def from_map_file(
cls, path: str | Path = DEFAULT_LEPCHA_LEGACY_MAP
) -> "LepchaLegacyConverter":
map_path = Path(path)
if not map_path.is_file():
raise DataValidationError(f"Lepcha legacy map does not exist: {map_path}")
# Join backslash line continuations, then strip comments per line.
raw = map_path.read_text(encoding="utf-8-sig")
raw = re.sub(r"\\\s*\n", " ", raw)
byte_classes: dict[str, tuple[int, ...]] = {}
uni_classes_ordered: dict[str, tuple[int, ...]] = {}
plain_classes: dict[str, frozenset[int]] = {}
byte_rules: list[tuple[tuple[int, ...], tuple[int, ...]]] = []
reorder_rules: list[_ReorderRule] = []
context_rule: tuple[int, frozenset[int], int] | None = None
current_pass = ""
byte_rule_lines: list[str] = []
for raw_line in raw.splitlines():
line = raw_line.split(";", 1)[0].strip()
if not line:
continue
lowered = line.lower()
if lowered.startswith("pass("):
current_pass = lowered
continue
if current_pass == "pass(byte_unicode)":
byte_match = _BYTECLASS_RE.match(line)
if byte_match:
byte_classes[byte_match.group(1).strip()] = _expand_byte_tokens(
byte_match.group(2)
)
continue
uni_match = _UNICLASS_RE.match(line)
if uni_match:
uni_classes_ordered[uni_match.group(1).strip()] = _expand_uni_tokens(
uni_match.group(2)
)
continue
byte_rule_lines.append(line)
continue
if current_pass == "pass(unicode)":
plain_match = _PLAINCLASS_RE.match(line)
if plain_match:
plain_classes[plain_match.group(1).strip()] = frozenset(
_expand_uni_tokens(plain_match.group(2))
)
continue
rule = cls._parse_reorder_rule(line)
if rule is not None:
reorder_rules.append(rule)
continue
# Second pass over Byte_Unicode rule lines so class rules can reference
# classes regardless of declaration order.
for line in byte_rule_lines:
ctx = _CONTEXT_RULE_RE.match(line)
if ctx:
cls_name = ctx.group(2).strip()
cls_bytes = byte_classes.get(cls_name)
if cls_bytes is None:
raise DataValidationError(
f"context rule references unknown byte class: {cls_name!r}"
)
# The context class names a *byte* class but the run-time check is
# against already-converted Unicode codepoints. Map the byte class
# to its Unicode equivalents via the paired UniClass.
cls_codepoints = uni_classes_ordered.get(cls_name)
if cls_codepoints is None:
raise DataValidationError(
f"context rule class {cls_name!r} has no UniClass"
)
context_rule = (
int(ctx.group(1), 16),
frozenset(cls_codepoints),
int(ctx.group(3), 16),
)
continue
class_rule = _CLASS_RULE_RE.match(line)
if class_rule:
left_name = class_rule.group(1).strip()
right_name = class_rule.group(2).strip()
left = byte_classes.get(left_name)
right = uni_classes_ordered.get(right_name)
if left is None:
raise DataValidationError(
f"class rule references unknown byte class: {left_name!r}"
)
if right is None:
raise DataValidationError(
f"class rule references unknown unicode class: {right_name!r}"
)
if len(left) != len(right):
raise DataValidationError(
f"class rule length mismatch for [{left_name}]>[{right_name}]: "
f"{len(left)} bytes vs {len(right)} codepoints"
)
for byte_value, codepoint in zip(left, right):
byte_rules.append(((byte_value,), (codepoint,)))
continue
if "<>" in line or ">" in line:
left_text, right_text = (
line.split("<>", 1) if "<>" in line else line.split(">", 1)
)
source = _BYTE_TOKEN_RE.findall(left_text)
target = _UNI_TOKEN_RE.findall(right_text)
if source and target:
byte_rules.append(
(
tuple(int(value, 16) for value in source),
tuple(int(value, 16) for value in target),
)
)
return cls(byte_rules, reorder_rules, plain_classes, context_rule)
@staticmethod
def _parse_reorder_rule(line: str) -> _ReorderRule | None:
if "<>" not in line:
return None
left, right = line.split("<>", 1)
slots = tuple(
(cls.strip(), var.strip())
for cls, var in _BOUND_CLASS_RE.findall(left)
)
output_vars = tuple(_VAR_REF_RE.findall(right))
if not slots or not output_vars:
return None
# Only keep well-formed permutation rules where every output var is bound.
bound = {var for _, var in slots}
if any(var not in bound for var in output_vars):
return None
return _ReorderRule(slots=slots, output_vars=output_vars)
# ----- Pass 1: Byte_Unicode -------------------------------------------------
def _apply_byte_pass(self, text: str) -> tuple[str, int, list[str]]:
output: list[str] = []
unmapped: list[str] = []
replacements = 0
index = 0
length = len(text)
while index < length:
# Context rule: bare 0x61 after a (previously emitted) dependent vowel
# becomes the independent vowel A (U+1C26) rather than dep. vowel AA.
if self._context_rule is not None:
trigger_byte, prev_classes, replacement = self._context_rule
if (
ord(text[index]) == trigger_byte
and output
and ord(output[-1]) in prev_classes
):
output.append(chr(replacement))
replacements += 1
index += 1
continue
matched = False
for source, target in self._byte_rules:
if self._matches(text, index, source):
output.extend(chr(value) for value in target)
replacements += 1
index += len(source)
matched = True
break
if matched:
continue
char = text[index]
output.append(char)
code = ord(char)
if code > 0x7F and not (LEPCHA_LO <= code <= LEPCHA_HI):
unmapped.append(f"U+{code:04X}")
index += 1
return "".join(output), replacements, unmapped
# ----- Pass 2: Unicode reorder ---------------------------------------------
def _apply_reorder_pass(self, text: str) -> str:
output: list[str] = []
index = 0
length = len(text)
while index < length:
applied = False
for rule in self._reorder_rules:
bound = self._match_reorder(text, index, rule)
if bound is not None:
output.extend(bound[var] for var in rule.output_vars)
index += len(rule.slots)
applied = True
break
if applied:
continue
output.append(text[index])
index += 1
return "".join(output)
def _match_reorder(
self, text: str, index: int, rule: _ReorderRule
) -> dict[str, str] | None:
if index + len(rule.slots) > len(text):
return None
bound: dict[str, str] = {}
for offset, (cls_name, var_name) in enumerate(rule.slots):
members = self._uni_classes.get(cls_name)
if members is None:
return None
ch = text[index + offset]
if ord(ch) not in members:
return None
bound[var_name] = ch
return bound
@staticmethod
def _matches(text: str, index: int, source: tuple[int, ...]) -> bool:
if index + len(source) > len(text):
return False
return all(ord(text[index + offset]) == code for offset, code in enumerate(source))
# ----- public --------------------------------------------------------------
def convert(self, text: str) -> LepchaLegacyConversion:
stage1, replacements, unmapped = self._apply_byte_pass(text)
stage2 = self._apply_reorder_pass(stage1)
converted = unicodedata.normalize("NFC", stage2)
return LepchaLegacyConversion(
legacy_text=text,
unicode_text=converted,
lepcha_char_count=len(_LEPCHA_CODEPOINT_RE.findall(converted)),
replacement_count=replacements,
unmapped_codepoints=sorted(set(unmapped)),
)
def convert_lepcha_legacy_text(
text: str, map_path: str | Path = DEFAULT_LEPCHA_LEGACY_MAP
) -> LepchaLegacyConversion:
return LepchaLegacyConverter.from_map_file(map_path).convert(text)