File size: 12,287 Bytes
d82db76 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 | """Convert Sikkim Herald Sunuwar (Jenticha / Koĩts) legacy font text to Unicode.
The Sunuwar edition of the Sikkim Herald ("Mukhia") is typeset in a legacy display
font that ships under two BaseFont names -- ``koits`` (the 2021-07-16 edition, a
Type0/Identity-H CID font) and ``kirat1`` (the 2023/2024 editions, byte-encoded
TrueType, WinAnsi). Each visible glyph is addressed by a single Latin byte; the
extractable text stream is therefore the raw legacy byte sequence, not Unicode.
There is no published byte->Unicode table for this layout. The legacy Kirat1 font
itself is documented in the Unicode proposal (L2/21-157R, Pandey 2021): "A digitized
font for the script named 'Kirat1' was developed by Shyan Kirat Rai. It is based upon
a non-Unicode encoding and is mapped to Latin letters." -- i.e. a display font where
each Latin byte directly draws a Sunuwar glyph; it is NOT the Keyman "Sunuwar
(Phonetic)" keyboard layout (the two agree on only 2/19 known bytes). The Sunuwar
block (U+11BC0-U+11BFF) is a 33-LETTER ALPHABET plus one auspicious sign (U+11BE1)
and 10 digits, with NO dependent vowel signs, so every legacy byte is a full letter,
digit, or punctuation mark.
The map below was derived by GLYPH-SHAPE IDENTITY: ``koits`` and ``kirat1`` were
proven to be the *same* embedded font (byte-for-byte identical outlines), so a single
map serves both. Each legacy byte's outline glyph was rendered and matched against the
44 Sunuwar Unicode codepoints in Noto Sans Sunuwar (U+11BC0-U+11BFF). The first pass
fixed 29 letter+digit bytes by isolated shape + printed crops. The second pass solved
the remaining 9 bytes as a one-to-one (bijective) assignment to the still-free
codepoints, using shape (IoU + chamfer) with a topological hole-count constraint,
adjudicated against the proposal's reference glyph chart, and validated by a
printed-ink single-glyph round-trip (the embedded Kirat1 outline equals printed ink).
That resolved 8 of the 9; one byte (``|``) is left flagged. See
``outputs/sunuwar-map-derivation/`` for the contact sheets, printed-crop verification,
context-match (``uncertain_context_match.png``), and round-trip images.
Confidence tiers are explicit:
* ``SUNUWAR_DIGITS`` -- all 10 digits, validated (byte '0'..'9' -> U+11BF0..U+11BF9
in order, clean top-1 shape match, round-trip IoU 0.53-0.78).
* ``SUNUWAR_LETTERS_CONFIRMED`` -- 27 letter bytes whose Unicode assignment is
confirmed by a printed-crop/printed-ink visual match and/or a clear round-trip
score with a decisive margin over alternatives. Includes the 8 bytes resolved in
the second pass (``v q x r u g h j``).
* ``SUNUWAR_LETTERS_UNCERTAIN`` -- the single byte (``|``) still unresolved. NOT
applied by default (``convert`` leaves it as-is and records it) so that no
unverified codepoint is committed to ground-truth output.
Punctuation bytes (``,`` ``:`` ``-`` ``(`` ``)`` ``\\`` ``.`` and the heavily-used
``=`` ``/`` which are danda-like marks in this layout) are passed through unchanged.
"""
from __future__ import annotations
import unicodedata
from dataclasses import dataclass, field
from typing import Any
from .errors import DataValidationError
# --- Digits: validated, byte '0'..'9' -> U+11BF0..U+11BF9 in order. ---
SUNUWAR_DIGITS: dict[str, str] = {str(i): chr(0x11BF0 + i) for i in range(10)}
# --- Letters: CONFIRMED tier (printed-crop and/or round-trip verified). ---
# byte -> Unicode codepoint (Noto glyph name in comment).
SUNUWAR_LETTERS_CONFIRMED: dict[str, str] = {
"{": chr(0x11BC3), # imarsunuwar (cross '+') round-trip IoU 0.74, printed-crop match
"}": chr(0x11BC2), # ekosunuwar ('土') round-trip IoU 0.55, printed-crop match
"z": chr(0x11BC9), # pipsunuwar (curl '9') printed-crop match (curl), margin 0.20
"A": chr(0x11BD6), # aalsunuwar ('Z'/zigzag) round-trip IoU 0.52, printed-crop match
"O": chr(0x11BD1), # otthisunuwar round-trip IoU 0.58, margin 0.14
"i": chr(0x11BCC), # carmisunuwar ('<') round-trip IoU 0.71, printed-crop match
"k": chr(0x11BDE), # tentusunuwar ('E'-stack) round-trip IoU 0.71, printed-crop match
"l": chr(0x11BDF), # thelesunuwar round-trip IoU 0.67, margin 0.17
"m": chr(0x11BC1), # taslasunuwar round-trip IoU 0.58, margin 0.08
"n": chr(0x11BD8), # tharisunuwar round-trip IoU 0.52, margin 0.07
"o": chr(0x11BC0), # devisunuwar ('刀' box) round-trip IoU 0.50, printed-crop match
"p": chr(0x11BCD), # nahsunuwar round-trip IoU 0.54, margin 0.10
"w": chr(0x11BD0), # loachasunuwar('Ш' comb) round-trip IoU 0.46, printed-crop match
"y": chr(0x11BDC), # shyersunuwar ('U') round-trip IoU 0.82, printed-crop match
"f": chr(0x11BDB), # khasunuwar ('Π' gate) round-trip IoU 0.46, printed-crop match
"s": chr(0x11BCE), # bursunuwar (cross+hook) round-trip IoU 0.54, printed-crop match
"a": chr(0x11BC8), # apphosunuwar (rev-'3' curl) round-trip IoU 0.51, printed-crop match
"t": chr(0x11BC7), # masunuwar ('Ш' comb) printed-crop match (comb=comb); n=1259
"e": chr(0x11BCB), # hamsosunuwar (cross+foot) printed-crop match (cross+hook); n=1006
# --- Below: the 8 bytes resolved by the second derivation pass. The Sunuwar block
# (U+11BC0-U+11BE0) is a 33-LETTER ALPHABET + 1 auspicious sign (U+11BE1) with NO
# dependent vowel signs (confirmed against the L2/21-157R Unicode proposal chart and
# the Keyman "Sunuwar (Phonetic)" .kmn LettersU store), so every uncertain byte is a
# full letter. The Keyman phonetic keyboard is a *different* layout from this Kirat1
# legacy display font (agree on only 2/19 bytes), so it cannot map the bytes directly;
# it only fixes each codepoint's canonical glyph identity. Resolution method: a
# one-to-one (bijective) assignment of the 9 uncertain bytes to the 15 still-free
# codepoints, scored by Kirat1-glyph-vs-Noto-glyph shape (IoU + chamfer) with a
# topological hole-count constraint, then adjudicated against the proposal's reference
# chart and validated by a printed-ink single-glyph round-trip (the Kirat1 embedded
# outline is byte-identical to printed ink, proven in pass 1). 'g'/'j' were confirmed
# on the clean embedded-glyph reference after slice-contamination ambiguity.
# Evidence images: outputs/sunuwar-map-derivation/{uncertain_context_match,
# uncertain_topk, all_shapes_grid, printink_belowfloor, disambig_clusters}.png
"v": chr(0x11BC4), # reusunuwar (N+diagonal) shape 0.545>=floor; bijection+context-match; n=765
"q": chr(0x11BE0), # klokosunuwar (bowl+stem) shape 0.567>=floor; bijection+context-match; n=579
"x": chr(0x11BD3), # varcasunuwar (N+hook) shape 0.522>=floor; bijection+context-match; n=97
"r": chr(0x11BD9), # pharsunuwar ('alpha') shape 0.521>=floor; hole-match 1=1; n=40
"u": chr(0x11BD4), # yatsunuwar (looped 'e') topology-locked (1 hole=loop); printed-ink 7/8 vote
"g": chr(0x11BD5), # avasunuwar (forked vert) embedded-glyph 0.454 decisive over gil; printed-ink
"h": chr(0x11BDA), # ngarsunuwar ('3') printed-ink 7/8 vote over donga('5'); n=180
"j": chr(0x11BCF), # jyahsunuwar (dagger '+') embedded-glyph 0.504 decisive (dagger=dagger); n=198
}
# --- Letters: UNCERTAIN tier. The best current guess is kept here for future
# verification but is NOT applied unless the caller opts in. ---
SUNUWAR_LETTERS_UNCERTAIN: dict[str, str] = {
# '|' (n=1079, the single most frequent uncertain byte) is the only byte that
# remains unresolved. Its Kirat1 glyph is a flowing open '2'/integral curl with NO
# internal loop (0 holes), which excludes the looped letters. Shape matching favors
# SHYELE/ʃ (U+11BD2, combo 0.501) over the next 0-hole candidate UTTHI/u (U+11BC5,
# 0.385) -- but its distribution argues the opposite: '|' is overwhelmingly
# word-medial/final (762 medial, 301 final, ends 302 words) and extremely frequent,
# the profile of a common VOWEL (utthi /u/), not a rare sibilant (shyele /ʃ/). Shape
# and phonology disagree and neither can be certified by round-trip, so '|' is left
# flagged. UNBLOCK: a native-reader transcription of any short Sikkim-Herald passage
# containing '|' (e.g. raw 't|o{:' or 'l}s t|v| q|e{ p|') to read off whether '|' is
# /u/ (utthi) or /ʃ/ (shyele). Best guess below is utthi on the phonology argument.
"|": chr(0x11BC5), # utthi? (or shyele U+11BD2) -- shape vs phonology conflict; needs a reader
}
# Punctuation / danda-like bytes: passed through unchanged (not Sunuwar letters).
SUNUWAR_PASSTHROUGH: frozenset[str] = frozenset(
{",", ":", "-", "(", ")", "\\", ".", "=", "/", "_", "<", "+", "]", "[", ";", "'", '"', "!", "?", "%"}
)
_SUNUWAR_BLOCK_LO = 0x11BC0
_SUNUWAR_BLOCK_HI = 0x11BFF
def _build_default_table() -> dict[str, str]:
table: dict[str, str] = {}
table.update(SUNUWAR_DIGITS)
table.update(SUNUWAR_LETTERS_CONFIRMED)
return table
@dataclass(frozen=True, slots=True)
class SunuwarLegacyConversion:
legacy_text: str
unicode_text: str
sunuwar_char_count: int
replacement_count: int
confirmed_byte_count: int
uncertain_bytes: list[str] = field(default_factory=list)
unmapped_bytes: list[str] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"legacy_text": self.legacy_text,
"unicode_text": self.unicode_text,
"sunuwar_char_count": self.sunuwar_char_count,
"replacement_count": self.replacement_count,
"confirmed_byte_count": self.confirmed_byte_count,
"uncertain_bytes": self.uncertain_bytes,
"unmapped_bytes": self.unmapped_bytes,
}
class SunuwarLegacyConverter:
"""Apply the derived Sunuwar legacy byte -> Unicode map.
By default only the CONFIRMED letters + digits are applied. Set
``apply_uncertain=True`` to additionally apply the best-guess UNCERTAIN map
(for exploration only -- never for ground-truth output).
"""
def __init__(self, apply_uncertain: bool = False) -> None:
table = _build_default_table()
if apply_uncertain:
table.update(SUNUWAR_LETTERS_UNCERTAIN)
if not table:
raise DataValidationError("Sunuwar legacy converter requires a non-empty map")
self._table = table
self._apply_uncertain = apply_uncertain
def convert(self, text: str) -> SunuwarLegacyConversion:
out: list[str] = []
replacements = 0
confirmed = 0
uncertain_seen: set[str] = set()
unmapped: set[str] = set()
for ch in text:
if ch == " " or ch == "\n" or ch == "\t":
out.append(ch)
continue
mapped = self._table.get(ch)
if mapped is not None:
out.append(mapped)
replacements += 1
if ch in SUNUWAR_LETTERS_CONFIRMED or ch in SUNUWAR_DIGITS:
confirmed += 1
continue
if ch in SUNUWAR_LETTERS_UNCERTAIN:
uncertain_seen.add(ch)
out.append(ch) # left untouched when not applying uncertain
continue
if ch in SUNUWAR_PASSTHROUGH:
out.append(ch)
continue
out.append(ch)
if ord(ch) > 0x7F and not (_SUNUWAR_BLOCK_LO <= ord(ch) <= _SUNUWAR_BLOCK_HI):
unmapped.add(ch)
converted = unicodedata.normalize("NFC", "".join(out))
sun_count = sum(1 for c in converted if _SUNUWAR_BLOCK_LO <= ord(c) <= _SUNUWAR_BLOCK_HI)
return SunuwarLegacyConversion(
legacy_text=text,
unicode_text=converted,
sunuwar_char_count=sun_count,
replacement_count=replacements,
confirmed_byte_count=confirmed,
uncertain_bytes=sorted(uncertain_seen),
unmapped_bytes=sorted(unmapped),
)
def convert_sunuwar_legacy_text(
text: str, apply_uncertain: bool = False
) -> SunuwarLegacyConversion:
return SunuwarLegacyConverter(apply_uncertain=apply_uncertain).convert(text)
|