"""Convert Sikkim Herald Sunuwar (Jenticha / Koĩts) legacy font text to Unicode. The Sunuwar edition of the Sikkim Herald ("Mukhia") is typeset in a legacy display font that ships under two BaseFont names -- ``koits`` (the 2021-07-16 edition, a Type0/Identity-H CID font) and ``kirat1`` (the 2023/2024 editions, byte-encoded TrueType, WinAnsi). Each visible glyph is addressed by a single Latin byte; the extractable text stream is therefore the raw legacy byte sequence, not Unicode. There is no published byte->Unicode table for this layout. The legacy Kirat1 font itself is documented in the Unicode proposal (L2/21-157R, Pandey 2021): "A digitized font for the script named 'Kirat1' was developed by Shyan Kirat Rai. It is based upon a non-Unicode encoding and is mapped to Latin letters." -- i.e. a display font where each Latin byte directly draws a Sunuwar glyph; it is NOT the Keyman "Sunuwar (Phonetic)" keyboard layout (the two agree on only 2/19 known bytes). The Sunuwar block (U+11BC0-U+11BFF) is a 33-LETTER ALPHABET plus one auspicious sign (U+11BE1) and 10 digits, with NO dependent vowel signs, so every legacy byte is a full letter, digit, or punctuation mark. The map below was derived by GLYPH-SHAPE IDENTITY: ``koits`` and ``kirat1`` were proven to be the *same* embedded font (byte-for-byte identical outlines), so a single map serves both. Each legacy byte's outline glyph was rendered and matched against the 44 Sunuwar Unicode codepoints in Noto Sans Sunuwar (U+11BC0-U+11BFF). The first pass fixed 29 letter+digit bytes by isolated shape + printed crops. The second pass solved the remaining 9 bytes as a one-to-one (bijective) assignment to the still-free codepoints, using shape (IoU + chamfer) with a topological hole-count constraint, adjudicated against the proposal's reference glyph chart, and validated by a printed-ink single-glyph round-trip (the embedded Kirat1 outline equals printed ink). That resolved 8 of the 9; one byte (``|``) is left flagged. See ``outputs/sunuwar-map-derivation/`` for the contact sheets, printed-crop verification, context-match (``uncertain_context_match.png``), and round-trip images. Confidence tiers are explicit: * ``SUNUWAR_DIGITS`` -- all 10 digits, validated (byte '0'..'9' -> U+11BF0..U+11BF9 in order, clean top-1 shape match, round-trip IoU 0.53-0.78). * ``SUNUWAR_LETTERS_CONFIRMED`` -- 27 letter bytes whose Unicode assignment is confirmed by a printed-crop/printed-ink visual match and/or a clear round-trip score with a decisive margin over alternatives. Includes the 8 bytes resolved in the second pass (``v q x r u g h j``). * ``SUNUWAR_LETTERS_UNCERTAIN`` -- the single byte (``|``) still unresolved. NOT applied by default (``convert`` leaves it as-is and records it) so that no unverified codepoint is committed to ground-truth output. Punctuation bytes (``,`` ``:`` ``-`` ``(`` ``)`` ``\\`` ``.`` and the heavily-used ``=`` ``/`` which are danda-like marks in this layout) are passed through unchanged. """ from __future__ import annotations import unicodedata from dataclasses import dataclass, field from typing import Any from .errors import DataValidationError # --- Digits: validated, byte '0'..'9' -> U+11BF0..U+11BF9 in order. --- SUNUWAR_DIGITS: dict[str, str] = {str(i): chr(0x11BF0 + i) for i in range(10)} # --- Letters: CONFIRMED tier (printed-crop and/or round-trip verified). --- # byte -> Unicode codepoint (Noto glyph name in comment). SUNUWAR_LETTERS_CONFIRMED: dict[str, str] = { "{": chr(0x11BC3), # imarsunuwar (cross '+') round-trip IoU 0.74, printed-crop match "}": chr(0x11BC2), # ekosunuwar ('土') round-trip IoU 0.55, printed-crop match "z": chr(0x11BC9), # pipsunuwar (curl '9') printed-crop match (curl), margin 0.20 "A": chr(0x11BD6), # aalsunuwar ('Z'/zigzag) round-trip IoU 0.52, printed-crop match "O": chr(0x11BD1), # otthisunuwar round-trip IoU 0.58, margin 0.14 "i": chr(0x11BCC), # carmisunuwar ('<') round-trip IoU 0.71, printed-crop match "k": chr(0x11BDE), # tentusunuwar ('E'-stack) round-trip IoU 0.71, printed-crop match "l": chr(0x11BDF), # thelesunuwar round-trip IoU 0.67, margin 0.17 "m": chr(0x11BC1), # taslasunuwar round-trip IoU 0.58, margin 0.08 "n": chr(0x11BD8), # tharisunuwar round-trip IoU 0.52, margin 0.07 "o": chr(0x11BC0), # devisunuwar ('刀' box) round-trip IoU 0.50, printed-crop match "p": chr(0x11BCD), # nahsunuwar round-trip IoU 0.54, margin 0.10 "w": chr(0x11BD0), # loachasunuwar('Ш' comb) round-trip IoU 0.46, printed-crop match "y": chr(0x11BDC), # shyersunuwar ('U') round-trip IoU 0.82, printed-crop match "f": chr(0x11BDB), # khasunuwar ('Π' gate) round-trip IoU 0.46, printed-crop match "s": chr(0x11BCE), # bursunuwar (cross+hook) round-trip IoU 0.54, printed-crop match "a": chr(0x11BC8), # apphosunuwar (rev-'3' curl) round-trip IoU 0.51, printed-crop match "t": chr(0x11BC7), # masunuwar ('Ш' comb) printed-crop match (comb=comb); n=1259 "e": chr(0x11BCB), # hamsosunuwar (cross+foot) printed-crop match (cross+hook); n=1006 # --- Below: the 8 bytes resolved by the second derivation pass. The Sunuwar block # (U+11BC0-U+11BE0) is a 33-LETTER ALPHABET + 1 auspicious sign (U+11BE1) with NO # dependent vowel signs (confirmed against the L2/21-157R Unicode proposal chart and # the Keyman "Sunuwar (Phonetic)" .kmn LettersU store), so every uncertain byte is a # full letter. The Keyman phonetic keyboard is a *different* layout from this Kirat1 # legacy display font (agree on only 2/19 bytes), so it cannot map the bytes directly; # it only fixes each codepoint's canonical glyph identity. Resolution method: a # one-to-one (bijective) assignment of the 9 uncertain bytes to the 15 still-free # codepoints, scored by Kirat1-glyph-vs-Noto-glyph shape (IoU + chamfer) with a # topological hole-count constraint, then adjudicated against the proposal's reference # chart and validated by a printed-ink single-glyph round-trip (the Kirat1 embedded # outline is byte-identical to printed ink, proven in pass 1). 'g'/'j' were confirmed # on the clean embedded-glyph reference after slice-contamination ambiguity. # Evidence images: outputs/sunuwar-map-derivation/{uncertain_context_match, # uncertain_topk, all_shapes_grid, printink_belowfloor, disambig_clusters}.png "v": chr(0x11BC4), # reusunuwar (N+diagonal) shape 0.545>=floor; bijection+context-match; n=765 "q": chr(0x11BE0), # klokosunuwar (bowl+stem) shape 0.567>=floor; bijection+context-match; n=579 "x": chr(0x11BD3), # varcasunuwar (N+hook) shape 0.522>=floor; bijection+context-match; n=97 "r": chr(0x11BD9), # pharsunuwar ('alpha') shape 0.521>=floor; hole-match 1=1; n=40 "u": chr(0x11BD4), # yatsunuwar (looped 'e') topology-locked (1 hole=loop); printed-ink 7/8 vote "g": chr(0x11BD5), # avasunuwar (forked vert) embedded-glyph 0.454 decisive over gil; printed-ink "h": chr(0x11BDA), # ngarsunuwar ('3') printed-ink 7/8 vote over donga('5'); n=180 "j": chr(0x11BCF), # jyahsunuwar (dagger '+') embedded-glyph 0.504 decisive (dagger=dagger); n=198 } # --- Letters: UNCERTAIN tier. The best current guess is kept here for future # verification but is NOT applied unless the caller opts in. --- SUNUWAR_LETTERS_UNCERTAIN: dict[str, str] = { # '|' (n=1079, the single most frequent uncertain byte) is the only byte that # remains unresolved. Its Kirat1 glyph is a flowing open '2'/integral curl with NO # internal loop (0 holes), which excludes the looped letters. Shape matching favors # SHYELE/ʃ (U+11BD2, combo 0.501) over the next 0-hole candidate UTTHI/u (U+11BC5, # 0.385) -- but its distribution argues the opposite: '|' is overwhelmingly # word-medial/final (762 medial, 301 final, ends 302 words) and extremely frequent, # the profile of a common VOWEL (utthi /u/), not a rare sibilant (shyele /ʃ/). Shape # and phonology disagree and neither can be certified by round-trip, so '|' is left # flagged. UNBLOCK: a native-reader transcription of any short Sikkim-Herald passage # containing '|' (e.g. raw 't|o{:' or 'l}s t|v| q|e{ p|') to read off whether '|' is # /u/ (utthi) or /ʃ/ (shyele). Best guess below is utthi on the phonology argument. "|": chr(0x11BC5), # utthi? (or shyele U+11BD2) -- shape vs phonology conflict; needs a reader } # Punctuation / danda-like bytes: passed through unchanged (not Sunuwar letters). SUNUWAR_PASSTHROUGH: frozenset[str] = frozenset( {",", ":", "-", "(", ")", "\\", ".", "=", "/", "_", "<", "+", "]", "[", ";", "'", '"', "!", "?", "%"} ) _SUNUWAR_BLOCK_LO = 0x11BC0 _SUNUWAR_BLOCK_HI = 0x11BFF def _build_default_table() -> dict[str, str]: table: dict[str, str] = {} table.update(SUNUWAR_DIGITS) table.update(SUNUWAR_LETTERS_CONFIRMED) return table @dataclass(frozen=True, slots=True) class SunuwarLegacyConversion: legacy_text: str unicode_text: str sunuwar_char_count: int replacement_count: int confirmed_byte_count: int uncertain_bytes: list[str] = field(default_factory=list) unmapped_bytes: list[str] = field(default_factory=list) def to_dict(self) -> dict[str, Any]: return { "legacy_text": self.legacy_text, "unicode_text": self.unicode_text, "sunuwar_char_count": self.sunuwar_char_count, "replacement_count": self.replacement_count, "confirmed_byte_count": self.confirmed_byte_count, "uncertain_bytes": self.uncertain_bytes, "unmapped_bytes": self.unmapped_bytes, } class SunuwarLegacyConverter: """Apply the derived Sunuwar legacy byte -> Unicode map. By default only the CONFIRMED letters + digits are applied. Set ``apply_uncertain=True`` to additionally apply the best-guess UNCERTAIN map (for exploration only -- never for ground-truth output). """ def __init__(self, apply_uncertain: bool = False) -> None: table = _build_default_table() if apply_uncertain: table.update(SUNUWAR_LETTERS_UNCERTAIN) if not table: raise DataValidationError("Sunuwar legacy converter requires a non-empty map") self._table = table self._apply_uncertain = apply_uncertain def convert(self, text: str) -> SunuwarLegacyConversion: out: list[str] = [] replacements = 0 confirmed = 0 uncertain_seen: set[str] = set() unmapped: set[str] = set() for ch in text: if ch == " " or ch == "\n" or ch == "\t": out.append(ch) continue mapped = self._table.get(ch) if mapped is not None: out.append(mapped) replacements += 1 if ch in SUNUWAR_LETTERS_CONFIRMED or ch in SUNUWAR_DIGITS: confirmed += 1 continue if ch in SUNUWAR_LETTERS_UNCERTAIN: uncertain_seen.add(ch) out.append(ch) # left untouched when not applying uncertain continue if ch in SUNUWAR_PASSTHROUGH: out.append(ch) continue out.append(ch) if ord(ch) > 0x7F and not (_SUNUWAR_BLOCK_LO <= ord(ch) <= _SUNUWAR_BLOCK_HI): unmapped.add(ch) converted = unicodedata.normalize("NFC", "".join(out)) sun_count = sum(1 for c in converted if _SUNUWAR_BLOCK_LO <= ord(c) <= _SUNUWAR_BLOCK_HI) return SunuwarLegacyConversion( legacy_text=text, unicode_text=converted, sunuwar_char_count=sun_count, replacement_count=replacements, confirmed_byte_count=confirmed, uncertain_bytes=sorted(uncertain_seen), unmapped_bytes=sorted(unmapped), ) def convert_sunuwar_legacy_text( text: str, apply_uncertain: bool = False ) -> SunuwarLegacyConversion: return SunuwarLegacyConverter(apply_uncertain=apply_uncertain).convert(text)