File size: 14,830 Bytes
f9bc0c1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
"""Convert canonical-new and Sikkim Herald legacy Kirat Rai text to Unicode.

SIL publishes a TECkit mapping for the canonical 2021 ``kirat rai font new``
encoding. Sikkim Herald PDFs use a different, older layout hidden behind per-PDF
font subsets and ASCII extraction values. Four initial subsets plus the two
2024/2025 subsets audited in 2026 share one stable old-to-new remap derived by
exact glyph-outline command and advance-width identity. The Herald converter
applies that premap before the SIL rules.

That map is expressed with TECkit ``ByteClass``/``UniClass`` declarations plus a
handful of explicit multi-byte ligature rules. A ``[class] > [class]`` rule maps
the byte at position *n* of the byte class to the codepoint at position *n* of the
paired Unicode class. This module parses that map natively (no ``teckit_compile``
dependency, which is unavailable on this machine) and applies the forward
Legacy->Unicode pass.

The two layouts must not be auto-detected from text: even Herald strings without
the formerly conspicuous ``f R x F I L`` bytes are globally permuted. Callers must
select the canonical-new or Herald converter explicitly. The later pair adds one
exact match, extracted ``T`` -> canonical ``Z`` -> U+16D6C KIRAT RAI VOWEL SIGN
EU. It also establishes that extracted ``C`` has an empty outline and that the
custom glyph stored under ``:`` is a visible double quotation mark. These
edition-specific findings are pinned by PDF/font hashes in the converter-layout
audit; they are not inferred from ASCII character names.
"""

from __future__ import annotations

import re
import unicodedata
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from .errors import DataValidationError

DEFAULT_KIRATRAI_LEGACY_MAP = Path(__file__).with_name("maps") / "kiratraifontnew.map"

# Bytes absent from SIL's canonical-new map. They remain a compatibility constant;
# in Herald PDFs these ASCII values belong to the separate permuted layout below.
UNMAPPED_LEGACY_BYTES: frozenset[str] = frozenset("fRxFIL\\")

# Sikkim Herald extracted ASCII -> canonical ``kiratraifontnew`` byte. Derived by
# exact RecordingPen outline-command and advance-width identity against the font
# embedded in Unicode proposal L2/22-043R. Shared by four PDF subsets.
KIRATRAI_HERALD_PREMAP: dict[str, str] = {
    "D": "q",
    "F": "g",
    "G": "P",
    "H": "j",
    "I": "W",
    "J": "J",
    "K": "Q",
    "L": "$",
    "O": "O",
    "R": "w",
    "S": "G",
    "T": "Z",
    "U": "o",
    "a": "k",
    "b": "A",
    "c": "D",
    "d": "K",
    "e": "m",
    "f": "N",
    "g": "s",
    "h": "a",
    "i": "v",
    "j": "b",
    "k": "r",
    "l": "i",
    "m": "c",
    "n": "p",
    "o": "n",
    "p": "t",
    "q": "d",
    "r": "e",
    "s": "C",
    "t": "h",
    "u": "u",
    "v": "B",
    "w": "l",
    "x": "T",
    "y": "y",
    "z": "z",
}

# Confirmed non-ink glyphs in the audited Herald layout. The two 2024/2025 font
# subsets give C an empty RecordingPen command stream; backslash was already proven
# blank in the earlier four-subset corpus. They normalize to an ordinary space.
KIRATRAI_HERALD_BLANK_GLYPHS: frozenset[str] = frozenset("\\C")

# The old font stores a visible opening/closing double-quotation glyph under the
# extracted ASCII colon code. Original-resolution review of the 2024-09-26 PDF,
# page 3, includes bboxes (75.239998, 566.760010, 81.175995, 584.615967) and
# (211.019241, 566.760010, 216.955246, 584.615967). This is a punctuation
# correction, not a script-letter inference.
KIRATRAI_HERALD_PUNCTUATION_MAP: dict[str, str] = {":": '"'}

# Confirmed literal values in the audited Herald PDFs.
KIRATRAI_HERALD_PASSTHROUGH: frozenset[str] = frozenset(" \t\r\n0123456789(),-/.;")

_KIRATRAI_CODEPOINT_RE = re.compile(r"[\U00016D40-\U00016D7F]")
_BYTE_TOKEN_RE = re.compile(r"0x([0-9A-Fa-f]{2})")
_UNI_TOKEN_RE = re.compile(r"U\+([0-9A-Fa-f]{4,6})")
_BYTECLASS_RE = re.compile(r"^ByteClass\s*\[([^\]]+)\]\s*=\s*\((.*)\)\s*$")
_UNICLASS_RE = re.compile(r"^UniClass\s*\[([^\]]+)\]\s*=\s*\((.*)\)\s*$")
_CLASS_RULE_RE = re.compile(r"^\[([^\]]+)\]\s*<?>\s*\[([^\]]+)\]\s*$")


def _expand_byte_tokens(body: str) -> tuple[int, ...]:
    """Expand a TECkit byte-class body into an ordered list of byte values.

    Supports both explicit ``0xNN`` tokens and ``0xNN .. 0xMM`` ranges.
    """
    values: list[int] = []
    tokens = re.split(r"\s+", body.strip())
    index = 0
    while index < len(tokens):
        token = tokens[index]
        if not token:
            index += 1
            continue
        if index + 2 < len(tokens) and tokens[index + 1] == "..":
            start = int(token, 16)
            end = int(tokens[index + 2], 16)
            if end < start:
                raise DataValidationError(f"invalid byte range in map: {token}..{tokens[index + 2]}")
            values.extend(range(start, end + 1))
            index += 3
            continue
        values.append(int(token, 16))
        index += 1
    return tuple(values)


def _expand_uni_tokens(body: str) -> tuple[int, ...]:
    """Expand a TECkit Unicode-class body into an ordered list of codepoints."""
    values: list[int] = []
    tokens = re.split(r"\s+", body.strip())
    index = 0
    while index < len(tokens):
        token = tokens[index]
        if not token:
            index += 1
            continue
        if index + 2 < len(tokens) and tokens[index + 1] == "..":
            start = int(_UNI_TOKEN_RE.match(token).group(1), 16)
            end = int(_UNI_TOKEN_RE.match(tokens[index + 2]).group(1), 16)
            if end < start:
                raise DataValidationError(f"invalid unicode range in map: {token}..{tokens[index + 2]}")
            values.extend(range(start, end + 1))
            index += 3
            continue
        match = _UNI_TOKEN_RE.match(token)
        if match is None:
            raise DataValidationError(f"unparseable unicode token in map: {token!r}")
        values.append(int(match.group(1), 16))
        index += 1
    return tuple(values)


@dataclass(frozen=True, slots=True)
class KiratRaiLegacyConversion:
    legacy_text: str
    unicode_text: str
    kiratrai_char_count: int
    replacement_count: int
    unmapped_codepoints: list[str]

    def to_dict(self) -> dict[str, Any]:
        return {
            "legacy_text": self.legacy_text,
            "unicode_text": self.unicode_text,
            "kiratrai_char_count": self.kiratrai_char_count,
            "replacement_count": self.replacement_count,
            "unmapped_codepoints": self.unmapped_codepoints,
        }


class KiratRaiLegacyConverter:
    """Native reader/applier for SIL's ``kiratraifontnew.map`` TECkit mapping.

    Rules are flattened to ``(source_bytes, target_codepoints)`` pairs and applied
    longest-match-first so that explicit multi-byte ligature rules (e.g. ``//`` ->
    double danda) take precedence over the single-byte class rules.
    """

    def __init__(self, rules: list[tuple[tuple[int, ...], tuple[int, ...]]]) -> None:
        if not rules:
            raise DataValidationError("Kirat Rai legacy converter requires at least one mapping rule")
        # De-duplicate while preserving the longest-source-first ordering.
        seen: set[tuple[int, ...]] = set()
        ordered: list[tuple[tuple[int, ...], tuple[int, ...]]] = []
        for source, target in sorted(rules, key=lambda item: len(item[0]), reverse=True):
            if source in seen:
                continue
            seen.add(source)
            ordered.append((source, target))
        self._rules = ordered

    @classmethod
    def from_map_file(cls, path: str | Path = DEFAULT_KIRATRAI_LEGACY_MAP) -> "KiratRaiLegacyConverter":
        map_path = Path(path)
        if not map_path.is_file():
            raise DataValidationError(f"Kirat Rai legacy map does not exist: {map_path}")
        byte_classes: dict[str, tuple[int, ...]] = {}
        uni_classes: dict[str, tuple[int, ...]] = {}
        rules: list[tuple[tuple[int, ...], tuple[int, ...]]] = []
        in_byte_pass = False
        # First parse class declarations; collect rule lines for a second pass so
        # class rules can reference classes regardless of declaration order.
        rule_lines: list[str] = []
        for raw_line in map_path.read_text(encoding="utf-8-sig").splitlines():
            line = raw_line.split(";", 1)[0].strip()
            if not line:
                continue
            lowered = line.lower()
            if lowered.startswith("pass("):
                in_byte_pass = lowered == "pass(byte_unicode)"
                continue
            if not in_byte_pass:
                continue
            byte_match = _BYTECLASS_RE.match(line)
            if byte_match:
                byte_classes[byte_match.group(1).strip()] = _expand_byte_tokens(byte_match.group(2))
                continue
            uni_match = _UNICLASS_RE.match(line)
            if uni_match:
                uni_classes[uni_match.group(1).strip()] = _expand_uni_tokens(uni_match.group(2))
                continue
            rule_lines.append(line)

        for line in rule_lines:
            class_rule = _CLASS_RULE_RE.match(line)
            if class_rule:
                left_name = class_rule.group(1).strip()
                right_name = class_rule.group(2).strip()
                left = byte_classes.get(left_name)
                right = uni_classes.get(right_name)
                if left is None:
                    raise DataValidationError(f"class rule references unknown byte class: {left_name!r}")
                if right is None:
                    raise DataValidationError(f"class rule references unknown unicode class: {right_name!r}")
                if len(left) != len(right):
                    raise DataValidationError(
                        f"class rule length mismatch for [{left_name}]>[{right_name}]: {len(left)} bytes vs {len(right)} codepoints"
                    )
                for byte_value, codepoint in zip(left, right):
                    rules.append(((byte_value,), (codepoint,)))
                continue
            if "<>" in line or ">" in line:
                left_text, right_text = line.split("<>", 1) if "<>" in line else line.split(">", 1)
                source = _BYTE_TOKEN_RE.findall(left_text)
                target = _UNI_TOKEN_RE.findall(right_text)
                if source and target:
                    rules.append(
                        (
                            tuple(int(value, 16) for value in source),
                            tuple(int(value, 16) for value in target),
                        )
                    )
        return cls(rules)

    def convert(self, text: str) -> KiratRaiLegacyConversion:
        output: list[str] = []
        unmapped: list[str] = []
        replacements = 0
        index = 0
        length = len(text)
        while index < length:
            matched = False
            for source, target in self._rules:
                if self._matches(text, index, source):
                    output.extend(chr(value) for value in target)
                    replacements += 1
                    index += len(source)
                    matched = True
                    break
            if matched:
                continue
            char = text[index]
            output.append(char)
            code = ord(char)
            if not (0x16D40 <= code <= 0x16D7F):
                unmapped.append(f"U+{code:04X}")
            index += 1
        converted = unicodedata.normalize("NFC", "".join(output))
        return KiratRaiLegacyConversion(
            legacy_text=text,
            unicode_text=converted,
            kiratrai_char_count=len(_KIRATRAI_CODEPOINT_RE.findall(converted)),
            replacement_count=replacements,
            unmapped_codepoints=sorted(set(unmapped)),
        )

    @staticmethod
    def _matches(text: str, index: int, source: tuple[int, ...]) -> bool:
        if index + len(source) > len(text):
            return False
        return all(ord(text[index + offset]) == code for offset, code in enumerate(source))


class KiratRaiHeraldConverter:
    """Convert the permuted Sikkim Herald layout through the canonical SIL map."""

    def __init__(self, canonical: KiratRaiLegacyConverter) -> None:
        self._canonical = canonical

    @classmethod
    def from_map_file(cls, path: str | Path = DEFAULT_KIRATRAI_LEGACY_MAP) -> "KiratRaiHeraldConverter":
        return cls(KiratRaiLegacyConverter.from_map_file(path))

    def convert(self, text: str) -> KiratRaiLegacyConversion:
        output: list[str] = []
        canonical_run: list[str] = []
        unmapped: set[str] = set()
        replacements = 0

        def flush() -> None:
            nonlocal replacements
            if not canonical_run:
                return
            result = self._canonical.convert("".join(canonical_run))
            output.append(result.unicode_text)
            replacements += result.replacement_count
            unmapped.update(result.unmapped_codepoints)
            canonical_run.clear()

        for char in text:
            remapped = KIRATRAI_HERALD_PREMAP.get(char)
            if remapped is not None:
                canonical_run.append(remapped)
                continue
            punctuation = KIRATRAI_HERALD_PUNCTUATION_MAP.get(char)
            if punctuation is not None:
                canonical_run.append(punctuation)
                continue
            if char in KIRATRAI_HERALD_BLANK_GLYPHS:
                canonical_run.append(" ")
                continue
            if char in KIRATRAI_HERALD_PASSTHROUGH:
                canonical_run.append(char)
                continue
            if 0x16D40 <= ord(char) <= 0x16D7F:
                flush()
                output.append(char)
                continue
            flush()
            output.append(char)
            unmapped.add(f"U+{ord(char):04X}")
        flush()

        converted = unicodedata.normalize("NFC", "".join(output))
        return KiratRaiLegacyConversion(
            legacy_text=text,
            unicode_text=converted,
            kiratrai_char_count=len(_KIRATRAI_CODEPOINT_RE.findall(converted)),
            replacement_count=replacements,
            unmapped_codepoints=sorted(unmapped),
        )


def convert_kiratrai_legacy_text(
    text: str, map_path: str | Path = DEFAULT_KIRATRAI_LEGACY_MAP
) -> KiratRaiLegacyConversion:
    return KiratRaiLegacyConverter.from_map_file(map_path).convert(text)


def convert_kiratrai_herald_text(
    text: str, map_path: str | Path = DEFAULT_KIRATRAI_LEGACY_MAP
) -> KiratRaiLegacyConversion:
    """Convert Sikkim Herald's older/permuted Kirat Rai layout to Unicode."""

    return KiratRaiHeraldConverter.from_map_file(map_path).convert(text)