File size: 9,190 Bytes
d82db76 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 | """Convert Limbu legacy Namdhinggo/Sirijonga text to Unicode Limbu."""
from __future__ import annotations
import json
import re
import unicodedata
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .errors import DataValidationError
DEFAULT_LIMBU_LEGACY_MAP = Path("data/mappings/limbu-legacy/Limbu.map")
_BYTE_RULE_RE = re.compile(r"0x([0-9A-Fa-f]{2})")
_UNICODE_RULE_RE = re.compile(r"U\+([0-9A-Fa-f]{4,6})")
_LIMBU_CODEPOINT_RE = re.compile(r"[\u1900-\u194F]")
_VOWEL_RANGE = range(0x1920, 0x1929)
_SUBJOINED_RANGE = range(0x1929, 0x192C)
_KEMPHRENG = "\u193A"
@dataclass(frozen=True, slots=True)
class LimbuLegacyConversion:
legacy_text: str
unicode_text: str
limbu_char_count: int
replacement_count: int
unmapped_codepoints: list[str]
def to_dict(self) -> dict[str, Any]:
return {
"legacy_text": self.legacy_text,
"unicode_text": self.unicode_text,
"limbu_char_count": self.limbu_char_count,
"replacement_count": self.replacement_count,
"unmapped_codepoints": self.unmapped_codepoints,
}
class LimbuLegacyConverter:
"""Small native reader for SIL's Limbu TECkit text map.
The upstream map is a TECkit map for Namdhinggo SIL L custom-encoded text.
This converter implements the forward Byte_Unicode rules used by the
Gorkhapatra PDF extraction path.
"""
def __init__(self, rules: list[tuple[tuple[int, ...], tuple[int, ...]]]) -> None:
if not rules:
raise DataValidationError("Limbu legacy converter requires at least one mapping rule")
# Stable longest-match-first ordering. For equal-length sources, the first
# rule seen wins; this lets an override map (parsed first) take precedence
# over a same-source rule in the canonical map without mutating the latter.
seen: set[tuple[int, ...]] = set()
ordered: list[tuple[tuple[int, ...], tuple[int, ...]]] = []
indexed = sorted(enumerate(rules), key=lambda item: (-len(item[1][0]), item[0]))
for _index, (src, tgt) in indexed:
if src in seen:
continue
seen.add(src)
ordered.append((src, tgt))
self._rules = ordered
@staticmethod
def _parse_byte_unicode_rules(
map_path: Path,
) -> list[tuple[tuple[int, ...], tuple[int, ...]]]:
rules: list[tuple[tuple[int, ...], tuple[int, ...]]] = []
in_byte_pass = False
for raw_line in map_path.read_text(encoding="utf-8").splitlines():
line = raw_line.split(";", 1)[0].strip()
if not line:
continue
if line.startswith("Pass("):
in_byte_pass = line == "Pass(Byte_Unicode)"
continue
if not in_byte_pass:
continue
if "<>" not in line and ">" not in line:
continue
left, right = line.split("<>", 1) if "<>" in line else line.split(">", 1)
source = tuple(int(value, 16) for value in _BYTE_RULE_RE.findall(left))
target = tuple(int(value, 16) for value in _UNICODE_RULE_RE.findall(right))
if source and target:
rules.append((source, target))
return rules
@classmethod
def from_map_file(
cls,
path: str | Path = DEFAULT_LIMBU_LEGACY_MAP,
*,
override_map_path: str | Path | None = None,
) -> "LimbuLegacyConverter":
map_path = Path(path)
if not map_path.is_file():
raise DataValidationError(f"Limbu legacy map does not exist: {map_path}")
# Override rules are parsed FIRST so that, on an equal-length source-byte
# tie, the override wins (see __init__: first rule seen per source wins).
rules: list[tuple[tuple[int, ...], tuple[int, ...]]] = []
if override_map_path is not None:
override_path = Path(override_map_path)
if not override_path.is_file():
raise DataValidationError(
f"Limbu legacy override map does not exist: {override_path}"
)
rules.extend(cls._parse_byte_unicode_rules(override_path))
rules.extend(cls._parse_byte_unicode_rules(map_path))
return cls(rules)
def convert(self, text: str) -> LimbuLegacyConversion:
output: list[str] = []
unmapped: list[str] = []
replacements = 0
index = 0
while index < len(text):
code = ord(text[index])
matched = False
for source, target in self._rules:
if self._matches(text, index, source):
output.extend(chr(value) for value in target)
replacements += 1
index += len(source)
matched = True
break
if matched:
continue
output.append(text[index])
if code > 0x7F and not (0x1900 <= code <= 0x194F):
unmapped.append(f"U+{code:04X}")
index += 1
converted = _reorder_limbu_vowels_and_subjoined("".join(output))
converted = unicodedata.normalize("NFC", converted)
return LimbuLegacyConversion(
legacy_text=text,
unicode_text=converted,
limbu_char_count=len(_LIMBU_CODEPOINT_RE.findall(converted)),
replacement_count=replacements,
unmapped_codepoints=sorted(set(unmapped)),
)
@staticmethod
def _matches(text: str, index: int, source: tuple[int, ...]) -> bool:
if index + len(source) > len(text):
return False
return all(ord(text[index + offset]) == code for offset, code in enumerate(source))
def convert_limbu_legacy_text(text: str, map_path: str | Path = DEFAULT_LIMBU_LEGACY_MAP) -> LimbuLegacyConversion:
return LimbuLegacyConverter.from_map_file(map_path).convert(text)
def convert_gorkhapatra_limbu_pdf_text(
pdf_text_json_path: str | Path,
output_dir: str | Path,
*,
map_path: str | Path = DEFAULT_LIMBU_LEGACY_MAP,
) -> dict[str, Any]:
source = Path(pdf_text_json_path)
if not source.is_file():
raise DataValidationError(f"PDF text JSON does not exist: {source}")
try:
payload = json.loads(source.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise DataValidationError(f"invalid PDF text JSON: {source}: {exc}") from exc
if not isinstance(payload, dict) or not isinstance(payload.get("spans"), list):
raise DataValidationError(f"PDF text JSON must contain a spans array: {source}")
converter = LimbuLegacyConverter.from_map_file(map_path)
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
rows: list[dict[str, Any]] = []
lines: list[str] = []
for span in payload["spans"]:
if not isinstance(span, dict) or not span.get("is_target_font"):
continue
conversion = converter.convert(str(span.get("raw_text") or ""))
row = {
"span_index": span.get("span_index"),
"block_index": span.get("block_index"),
"line_index": span.get("line_index"),
"font": span.get("font"),
"bbox": span.get("bbox"),
**conversion.to_dict(),
}
rows.append(row)
if conversion.unicode_text.strip():
lines.append(conversion.unicode_text.rstrip())
unicode_text = "\n".join(lines)
result = {
"source_pdf_text_json": str(source),
"map_path": str(Path(map_path)),
"sample_id": payload.get("sample_id"),
"candidate_language": payload.get("candidate_language"),
"pdf_page_number": payload.get("pdf_page_number"),
"target_span_count": len(rows),
"unicode_text": unicode_text,
"limbu_char_count": len(_LIMBU_CODEPOINT_RE.findall(unicode_text)),
"rows": rows,
}
sample_id = str(payload.get("sample_id") or source.stem)
json_path = out / f"{sample_id}.limbu-unicode.json"
text_path = out / f"{sample_id}.limbu-unicode.txt"
json_path.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
text_path.write_text(unicode_text + "\n", encoding="utf-8")
result["json_path"] = str(json_path)
result["text_path"] = str(text_path)
return result
def _reorder_limbu_vowels_and_subjoined(text: str) -> str:
chars = list(text)
index = 0
while index < len(chars) - 1:
current = ord(chars[index])
nxt = ord(chars[index + 1])
if current in _VOWEL_RANGE and nxt in _SUBJOINED_RANGE:
chars[index], chars[index + 1] = chars[index + 1], chars[index]
index += 2
continue
if index < len(chars) - 2 and current in _VOWEL_RANGE and chars[index + 1] == _KEMPHRENG and ord(chars[index + 2]) in _SUBJOINED_RANGE:
vowel = chars[index]
subjoined = chars[index + 2]
chars[index : index + 3] = [subjoined, vowel, _KEMPHRENG]
index += 3
continue
index += 1
return "".join(chars)
|