Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/src/ocrtech/sunuwar_legacy.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 12.3 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/sunuwar_legacy.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/src/ocrtech/sunuwar_legacy.py
-
curl -L -o sunuwar_legacy.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/sunuwar_legacy.py
12.3 kB
| """Convert Sikkim Herald Sunuwar (Jenticha / Koĩts) legacy font text to Unicode. | |
| The Sunuwar edition of the Sikkim Herald ("Mukhia") is typeset in a legacy display | |
| font that ships under two BaseFont names -- ``koits`` (the 2021-07-16 edition, a | |
| Type0/Identity-H CID font) and ``kirat1`` (the 2023/2024 editions, byte-encoded | |
| TrueType, WinAnsi). Each visible glyph is addressed by a single Latin byte; the | |
| extractable text stream is therefore the raw legacy byte sequence, not Unicode. | |
| There is no published byte->Unicode table for this layout. The legacy Kirat1 font | |
| itself is documented in the Unicode proposal (L2/21-157R, Pandey 2021): "A digitized | |
| font for the script named 'Kirat1' was developed by Shyan Kirat Rai. It is based upon | |
| a non-Unicode encoding and is mapped to Latin letters." -- i.e. a display font where | |
| each Latin byte directly draws a Sunuwar glyph; it is NOT the Keyman "Sunuwar | |
| (Phonetic)" keyboard layout (the two agree on only 2/19 known bytes). The Sunuwar | |
| block (U+11BC0-U+11BFF) is a 33-LETTER ALPHABET plus one auspicious sign (U+11BE1) | |
| and 10 digits, with NO dependent vowel signs, so every legacy byte is a full letter, | |
| digit, or punctuation mark. | |
| The map below was derived by GLYPH-SHAPE IDENTITY: ``koits`` and ``kirat1`` were | |
| proven to be the *same* embedded font (byte-for-byte identical outlines), so a single | |
| map serves both. Each legacy byte's outline glyph was rendered and matched against the | |
| 44 Sunuwar Unicode codepoints in Noto Sans Sunuwar (U+11BC0-U+11BFF). The first pass | |
| fixed 29 letter+digit bytes by isolated shape + printed crops. The second pass solved | |
| the remaining 9 bytes as a one-to-one (bijective) assignment to the still-free | |
| codepoints, using shape (IoU + chamfer) with a topological hole-count constraint, | |
| adjudicated against the proposal's reference glyph chart, and validated by a | |
| printed-ink single-glyph round-trip (the embedded Kirat1 outline equals printed ink). | |
| That resolved 8 of the 9; one byte (``|``) is left flagged. See | |
| ``outputs/sunuwar-map-derivation/`` for the contact sheets, printed-crop verification, | |
| context-match (``uncertain_context_match.png``), and round-trip images. | |
| Confidence tiers are explicit: | |
| * ``SUNUWAR_DIGITS`` -- all 10 digits, validated (byte '0'..'9' -> U+11BF0..U+11BF9 | |
| in order, clean top-1 shape match, round-trip IoU 0.53-0.78). | |
| * ``SUNUWAR_LETTERS_CONFIRMED`` -- 27 letter bytes whose Unicode assignment is | |
| confirmed by a printed-crop/printed-ink visual match and/or a clear round-trip | |
| score with a decisive margin over alternatives. Includes the 8 bytes resolved in | |
| the second pass (``v q x r u g h j``). | |
| * ``SUNUWAR_LETTERS_UNCERTAIN`` -- the single byte (``|``) still unresolved. NOT | |
| applied by default (``convert`` leaves it as-is and records it) so that no | |
| unverified codepoint is committed to ground-truth output. | |
| Punctuation bytes (``,`` ``:`` ``-`` ``(`` ``)`` ``\\`` ``.`` and the heavily-used | |
| ``=`` ``/`` which are danda-like marks in this layout) are passed through unchanged. | |
| """ | |
| from __future__ import annotations | |
| import unicodedata | |
| from dataclasses import dataclass, field | |
| from typing import Any | |
| from .errors import DataValidationError | |
| # --- Digits: validated, byte '0'..'9' -> U+11BF0..U+11BF9 in order. --- | |
| SUNUWAR_DIGITS: dict[str, str] = {str(i): chr(0x11BF0 + i) for i in range(10)} | |
| # --- Letters: CONFIRMED tier (printed-crop and/or round-trip verified). --- | |
| # byte -> Unicode codepoint (Noto glyph name in comment). | |
| SUNUWAR_LETTERS_CONFIRMED: dict[str, str] = { | |
| "{": chr(0x11BC3), # imarsunuwar (cross '+') round-trip IoU 0.74, printed-crop match | |
| "}": chr(0x11BC2), # ekosunuwar ('土') round-trip IoU 0.55, printed-crop match | |
| "z": chr(0x11BC9), # pipsunuwar (curl '9') printed-crop match (curl), margin 0.20 | |
| "A": chr(0x11BD6), # aalsunuwar ('Z'/zigzag) round-trip IoU 0.52, printed-crop match | |
| "O": chr(0x11BD1), # otthisunuwar round-trip IoU 0.58, margin 0.14 | |
| "i": chr(0x11BCC), # carmisunuwar ('<') round-trip IoU 0.71, printed-crop match | |
| "k": chr(0x11BDE), # tentusunuwar ('E'-stack) round-trip IoU 0.71, printed-crop match | |
| "l": chr(0x11BDF), # thelesunuwar round-trip IoU 0.67, margin 0.17 | |
| "m": chr(0x11BC1), # taslasunuwar round-trip IoU 0.58, margin 0.08 | |
| "n": chr(0x11BD8), # tharisunuwar round-trip IoU 0.52, margin 0.07 | |
| "o": chr(0x11BC0), # devisunuwar ('刀' box) round-trip IoU 0.50, printed-crop match | |
| "p": chr(0x11BCD), # nahsunuwar round-trip IoU 0.54, margin 0.10 | |
| "w": chr(0x11BD0), # loachasunuwar('Ш' comb) round-trip IoU 0.46, printed-crop match | |
| "y": chr(0x11BDC), # shyersunuwar ('U') round-trip IoU 0.82, printed-crop match | |
| "f": chr(0x11BDB), # khasunuwar ('Π' gate) round-trip IoU 0.46, printed-crop match | |
| "s": chr(0x11BCE), # bursunuwar (cross+hook) round-trip IoU 0.54, printed-crop match | |
| "a": chr(0x11BC8), # apphosunuwar (rev-'3' curl) round-trip IoU 0.51, printed-crop match | |
| "t": chr(0x11BC7), # masunuwar ('Ш' comb) printed-crop match (comb=comb); n=1259 | |
| "e": chr(0x11BCB), # hamsosunuwar (cross+foot) printed-crop match (cross+hook); n=1006 | |
| # --- Below: the 8 bytes resolved by the second derivation pass. The Sunuwar block | |
| # (U+11BC0-U+11BE0) is a 33-LETTER ALPHABET + 1 auspicious sign (U+11BE1) with NO | |
| # dependent vowel signs (confirmed against the L2/21-157R Unicode proposal chart and | |
| # the Keyman "Sunuwar (Phonetic)" .kmn LettersU store), so every uncertain byte is a | |
| # full letter. The Keyman phonetic keyboard is a *different* layout from this Kirat1 | |
| # legacy display font (agree on only 2/19 bytes), so it cannot map the bytes directly; | |
| # it only fixes each codepoint's canonical glyph identity. Resolution method: a | |
| # one-to-one (bijective) assignment of the 9 uncertain bytes to the 15 still-free | |
| # codepoints, scored by Kirat1-glyph-vs-Noto-glyph shape (IoU + chamfer) with a | |
| # topological hole-count constraint, then adjudicated against the proposal's reference | |
| # chart and validated by a printed-ink single-glyph round-trip (the Kirat1 embedded | |
| # outline is byte-identical to printed ink, proven in pass 1). 'g'/'j' were confirmed | |
| # on the clean embedded-glyph reference after slice-contamination ambiguity. | |
| # Evidence images: outputs/sunuwar-map-derivation/{uncertain_context_match, | |
| # uncertain_topk, all_shapes_grid, printink_belowfloor, disambig_clusters}.png | |
| "v": chr(0x11BC4), # reusunuwar (N+diagonal) shape 0.545>=floor; bijection+context-match; n=765 | |
| "q": chr(0x11BE0), # klokosunuwar (bowl+stem) shape 0.567>=floor; bijection+context-match; n=579 | |
| "x": chr(0x11BD3), # varcasunuwar (N+hook) shape 0.522>=floor; bijection+context-match; n=97 | |
| "r": chr(0x11BD9), # pharsunuwar ('alpha') shape 0.521>=floor; hole-match 1=1; n=40 | |
| "u": chr(0x11BD4), # yatsunuwar (looped 'e') topology-locked (1 hole=loop); printed-ink 7/8 vote | |
| "g": chr(0x11BD5), # avasunuwar (forked vert) embedded-glyph 0.454 decisive over gil; printed-ink | |
| "h": chr(0x11BDA), # ngarsunuwar ('3') printed-ink 7/8 vote over donga('5'); n=180 | |
| "j": chr(0x11BCF), # jyahsunuwar (dagger '+') embedded-glyph 0.504 decisive (dagger=dagger); n=198 | |
| } | |
| # --- Letters: UNCERTAIN tier. The best current guess is kept here for future | |
| # verification but is NOT applied unless the caller opts in. --- | |
| SUNUWAR_LETTERS_UNCERTAIN: dict[str, str] = { | |
| # '|' (n=1079, the single most frequent uncertain byte) is the only byte that | |
| # remains unresolved. Its Kirat1 glyph is a flowing open '2'/integral curl with NO | |
| # internal loop (0 holes), which excludes the looped letters. Shape matching favors | |
| # SHYELE/ʃ (U+11BD2, combo 0.501) over the next 0-hole candidate UTTHI/u (U+11BC5, | |
| # 0.385) -- but its distribution argues the opposite: '|' is overwhelmingly | |
| # word-medial/final (762 medial, 301 final, ends 302 words) and extremely frequent, | |
| # the profile of a common VOWEL (utthi /u/), not a rare sibilant (shyele /ʃ/). Shape | |
| # and phonology disagree and neither can be certified by round-trip, so '|' is left | |
| # flagged. UNBLOCK: a native-reader transcription of any short Sikkim-Herald passage | |
| # containing '|' (e.g. raw 't|o{:' or 'l}s t|v| q|e{ p|') to read off whether '|' is | |
| # /u/ (utthi) or /ʃ/ (shyele). Best guess below is utthi on the phonology argument. | |
| "|": chr(0x11BC5), # utthi? (or shyele U+11BD2) -- shape vs phonology conflict; needs a reader | |
| } | |
| # Punctuation / danda-like bytes: passed through unchanged (not Sunuwar letters). | |
| SUNUWAR_PASSTHROUGH: frozenset[str] = frozenset( | |
| {",", ":", "-", "(", ")", "\\", ".", "=", "/", "_", "<", "+", "]", "[", ";", "'", '"', "!", "?", "%"} | |
| ) | |
| _SUNUWAR_BLOCK_LO = 0x11BC0 | |
| _SUNUWAR_BLOCK_HI = 0x11BFF | |
| def _build_default_table() -> dict[str, str]: | |
| table: dict[str, str] = {} | |
| table.update(SUNUWAR_DIGITS) | |
| table.update(SUNUWAR_LETTERS_CONFIRMED) | |
| return table | |
| class SunuwarLegacyConversion: | |
| legacy_text: str | |
| unicode_text: str | |
| sunuwar_char_count: int | |
| replacement_count: int | |
| confirmed_byte_count: int | |
| uncertain_bytes: list[str] = field(default_factory=list) | |
| unmapped_bytes: list[str] = field(default_factory=list) | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "legacy_text": self.legacy_text, | |
| "unicode_text": self.unicode_text, | |
| "sunuwar_char_count": self.sunuwar_char_count, | |
| "replacement_count": self.replacement_count, | |
| "confirmed_byte_count": self.confirmed_byte_count, | |
| "uncertain_bytes": self.uncertain_bytes, | |
| "unmapped_bytes": self.unmapped_bytes, | |
| } | |
| class SunuwarLegacyConverter: | |
| """Apply the derived Sunuwar legacy byte -> Unicode map. | |
| By default only the CONFIRMED letters + digits are applied. Set | |
| ``apply_uncertain=True`` to additionally apply the best-guess UNCERTAIN map | |
| (for exploration only -- never for ground-truth output). | |
| """ | |
| def __init__(self, apply_uncertain: bool = False) -> None: | |
| table = _build_default_table() | |
| if apply_uncertain: | |
| table.update(SUNUWAR_LETTERS_UNCERTAIN) | |
| if not table: | |
| raise DataValidationError("Sunuwar legacy converter requires a non-empty map") | |
| self._table = table | |
| self._apply_uncertain = apply_uncertain | |
| def convert(self, text: str) -> SunuwarLegacyConversion: | |
| out: list[str] = [] | |
| replacements = 0 | |
| confirmed = 0 | |
| uncertain_seen: set[str] = set() | |
| unmapped: set[str] = set() | |
| for ch in text: | |
| if ch == " " or ch == "\n" or ch == "\t": | |
| out.append(ch) | |
| continue | |
| mapped = self._table.get(ch) | |
| if mapped is not None: | |
| out.append(mapped) | |
| replacements += 1 | |
| if ch in SUNUWAR_LETTERS_CONFIRMED or ch in SUNUWAR_DIGITS: | |
| confirmed += 1 | |
| continue | |
| if ch in SUNUWAR_LETTERS_UNCERTAIN: | |
| uncertain_seen.add(ch) | |
| out.append(ch) # left untouched when not applying uncertain | |
| continue | |
| if ch in SUNUWAR_PASSTHROUGH: | |
| out.append(ch) | |
| continue | |
| out.append(ch) | |
| if ord(ch) > 0x7F and not (_SUNUWAR_BLOCK_LO <= ord(ch) <= _SUNUWAR_BLOCK_HI): | |
| unmapped.add(ch) | |
| converted = unicodedata.normalize("NFC", "".join(out)) | |
| sun_count = sum(1 for c in converted if _SUNUWAR_BLOCK_LO <= ord(c) <= _SUNUWAR_BLOCK_HI) | |
| return SunuwarLegacyConversion( | |
| legacy_text=text, | |
| unicode_text=converted, | |
| sunuwar_char_count=sun_count, | |
| replacement_count=replacements, | |
| confirmed_byte_count=confirmed, | |
| uncertain_bytes=sorted(uncertain_seen), | |
| unmapped_bytes=sorted(unmapped), | |
| ) | |
| def convert_sunuwar_legacy_text( | |
| text: str, apply_uncertain: bool = False | |
| ) -> SunuwarLegacyConversion: | |
| return SunuwarLegacyConverter(apply_uncertain=apply_uncertain).convert(text) | |