Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/src/ocrtech/limbu_legacy.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 9.19 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/limbu_legacy.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/src/ocrtech/limbu_legacy.py
-
curl -L -o limbu_legacy.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/limbu_legacy.py
9.19 kB
| """Convert Limbu legacy Namdhinggo/Sirijonga text to Unicode Limbu.""" | |
| from __future__ import annotations | |
| import json | |
| import re | |
| import unicodedata | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any | |
| from .errors import DataValidationError | |
| DEFAULT_LIMBU_LEGACY_MAP = Path("data/mappings/limbu-legacy/Limbu.map") | |
| _BYTE_RULE_RE = re.compile(r"0x([0-9A-Fa-f]{2})") | |
| _UNICODE_RULE_RE = re.compile(r"U\+([0-9A-Fa-f]{4,6})") | |
| _LIMBU_CODEPOINT_RE = re.compile(r"[\u1900-\u194F]") | |
| _VOWEL_RANGE = range(0x1920, 0x1929) | |
| _SUBJOINED_RANGE = range(0x1929, 0x192C) | |
| _KEMPHRENG = "\u193A" | |
| class LimbuLegacyConversion: | |
| legacy_text: str | |
| unicode_text: str | |
| limbu_char_count: int | |
| replacement_count: int | |
| unmapped_codepoints: list[str] | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "legacy_text": self.legacy_text, | |
| "unicode_text": self.unicode_text, | |
| "limbu_char_count": self.limbu_char_count, | |
| "replacement_count": self.replacement_count, | |
| "unmapped_codepoints": self.unmapped_codepoints, | |
| } | |
| class LimbuLegacyConverter: | |
| """Small native reader for SIL's Limbu TECkit text map. | |
| The upstream map is a TECkit map for Namdhinggo SIL L custom-encoded text. | |
| This converter implements the forward Byte_Unicode rules used by the | |
| Gorkhapatra PDF extraction path. | |
| """ | |
| def __init__(self, rules: list[tuple[tuple[int, ...], tuple[int, ...]]]) -> None: | |
| if not rules: | |
| raise DataValidationError("Limbu legacy converter requires at least one mapping rule") | |
| # Stable longest-match-first ordering. For equal-length sources, the first | |
| # rule seen wins; this lets an override map (parsed first) take precedence | |
| # over a same-source rule in the canonical map without mutating the latter. | |
| seen: set[tuple[int, ...]] = set() | |
| ordered: list[tuple[tuple[int, ...], tuple[int, ...]]] = [] | |
| indexed = sorted(enumerate(rules), key=lambda item: (-len(item[1][0]), item[0])) | |
| for _index, (src, tgt) in indexed: | |
| if src in seen: | |
| continue | |
| seen.add(src) | |
| ordered.append((src, tgt)) | |
| self._rules = ordered | |
| def _parse_byte_unicode_rules( | |
| map_path: Path, | |
| ) -> list[tuple[tuple[int, ...], tuple[int, ...]]]: | |
| rules: list[tuple[tuple[int, ...], tuple[int, ...]]] = [] | |
| in_byte_pass = False | |
| for raw_line in map_path.read_text(encoding="utf-8").splitlines(): | |
| line = raw_line.split(";", 1)[0].strip() | |
| if not line: | |
| continue | |
| if line.startswith("Pass("): | |
| in_byte_pass = line == "Pass(Byte_Unicode)" | |
| continue | |
| if not in_byte_pass: | |
| continue | |
| if "<>" not in line and ">" not in line: | |
| continue | |
| left, right = line.split("<>", 1) if "<>" in line else line.split(">", 1) | |
| source = tuple(int(value, 16) for value in _BYTE_RULE_RE.findall(left)) | |
| target = tuple(int(value, 16) for value in _UNICODE_RULE_RE.findall(right)) | |
| if source and target: | |
| rules.append((source, target)) | |
| return rules | |
| def from_map_file( | |
| cls, | |
| path: str | Path = DEFAULT_LIMBU_LEGACY_MAP, | |
| *, | |
| override_map_path: str | Path | None = None, | |
| ) -> "LimbuLegacyConverter": | |
| map_path = Path(path) | |
| if not map_path.is_file(): | |
| raise DataValidationError(f"Limbu legacy map does not exist: {map_path}") | |
| # Override rules are parsed FIRST so that, on an equal-length source-byte | |
| # tie, the override wins (see __init__: first rule seen per source wins). | |
| rules: list[tuple[tuple[int, ...], tuple[int, ...]]] = [] | |
| if override_map_path is not None: | |
| override_path = Path(override_map_path) | |
| if not override_path.is_file(): | |
| raise DataValidationError( | |
| f"Limbu legacy override map does not exist: {override_path}" | |
| ) | |
| rules.extend(cls._parse_byte_unicode_rules(override_path)) | |
| rules.extend(cls._parse_byte_unicode_rules(map_path)) | |
| return cls(rules) | |
| def convert(self, text: str) -> LimbuLegacyConversion: | |
| output: list[str] = [] | |
| unmapped: list[str] = [] | |
| replacements = 0 | |
| index = 0 | |
| while index < len(text): | |
| code = ord(text[index]) | |
| matched = False | |
| for source, target in self._rules: | |
| if self._matches(text, index, source): | |
| output.extend(chr(value) for value in target) | |
| replacements += 1 | |
| index += len(source) | |
| matched = True | |
| break | |
| if matched: | |
| continue | |
| output.append(text[index]) | |
| if code > 0x7F and not (0x1900 <= code <= 0x194F): | |
| unmapped.append(f"U+{code:04X}") | |
| index += 1 | |
| converted = _reorder_limbu_vowels_and_subjoined("".join(output)) | |
| converted = unicodedata.normalize("NFC", converted) | |
| return LimbuLegacyConversion( | |
| legacy_text=text, | |
| unicode_text=converted, | |
| limbu_char_count=len(_LIMBU_CODEPOINT_RE.findall(converted)), | |
| replacement_count=replacements, | |
| unmapped_codepoints=sorted(set(unmapped)), | |
| ) | |
| def _matches(text: str, index: int, source: tuple[int, ...]) -> bool: | |
| if index + len(source) > len(text): | |
| return False | |
| return all(ord(text[index + offset]) == code for offset, code in enumerate(source)) | |
| def convert_limbu_legacy_text(text: str, map_path: str | Path = DEFAULT_LIMBU_LEGACY_MAP) -> LimbuLegacyConversion: | |
| return LimbuLegacyConverter.from_map_file(map_path).convert(text) | |
| def convert_gorkhapatra_limbu_pdf_text( | |
| pdf_text_json_path: str | Path, | |
| output_dir: str | Path, | |
| *, | |
| map_path: str | Path = DEFAULT_LIMBU_LEGACY_MAP, | |
| ) -> dict[str, Any]: | |
| source = Path(pdf_text_json_path) | |
| if not source.is_file(): | |
| raise DataValidationError(f"PDF text JSON does not exist: {source}") | |
| try: | |
| payload = json.loads(source.read_text(encoding="utf-8")) | |
| except json.JSONDecodeError as exc: | |
| raise DataValidationError(f"invalid PDF text JSON: {source}: {exc}") from exc | |
| if not isinstance(payload, dict) or not isinstance(payload.get("spans"), list): | |
| raise DataValidationError(f"PDF text JSON must contain a spans array: {source}") | |
| converter = LimbuLegacyConverter.from_map_file(map_path) | |
| out = Path(output_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| rows: list[dict[str, Any]] = [] | |
| lines: list[str] = [] | |
| for span in payload["spans"]: | |
| if not isinstance(span, dict) or not span.get("is_target_font"): | |
| continue | |
| conversion = converter.convert(str(span.get("raw_text") or "")) | |
| row = { | |
| "span_index": span.get("span_index"), | |
| "block_index": span.get("block_index"), | |
| "line_index": span.get("line_index"), | |
| "font": span.get("font"), | |
| "bbox": span.get("bbox"), | |
| **conversion.to_dict(), | |
| } | |
| rows.append(row) | |
| if conversion.unicode_text.strip(): | |
| lines.append(conversion.unicode_text.rstrip()) | |
| unicode_text = "\n".join(lines) | |
| result = { | |
| "source_pdf_text_json": str(source), | |
| "map_path": str(Path(map_path)), | |
| "sample_id": payload.get("sample_id"), | |
| "candidate_language": payload.get("candidate_language"), | |
| "pdf_page_number": payload.get("pdf_page_number"), | |
| "target_span_count": len(rows), | |
| "unicode_text": unicode_text, | |
| "limbu_char_count": len(_LIMBU_CODEPOINT_RE.findall(unicode_text)), | |
| "rows": rows, | |
| } | |
| sample_id = str(payload.get("sample_id") or source.stem) | |
| json_path = out / f"{sample_id}.limbu-unicode.json" | |
| text_path = out / f"{sample_id}.limbu-unicode.txt" | |
| json_path.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| text_path.write_text(unicode_text + "\n", encoding="utf-8") | |
| result["json_path"] = str(json_path) | |
| result["text_path"] = str(text_path) | |
| return result | |
| def _reorder_limbu_vowels_and_subjoined(text: str) -> str: | |
| chars = list(text) | |
| index = 0 | |
| while index < len(chars) - 1: | |
| current = ord(chars[index]) | |
| nxt = ord(chars[index + 1]) | |
| if current in _VOWEL_RANGE and nxt in _SUBJOINED_RANGE: | |
| chars[index], chars[index + 1] = chars[index + 1], chars[index] | |
| index += 2 | |
| continue | |
| if index < len(chars) - 2 and current in _VOWEL_RANGE and chars[index + 1] == _KEMPHRENG and ord(chars[index + 2]) in _SUBJOINED_RANGE: | |
| vowel = chars[index] | |
| subjoined = chars[index + 2] | |
| chars[index : index + 3] = [subjoined, vowel, _KEMPHRENG] | |
| index += 3 | |
| continue | |
| index += 1 | |
| return "".join(chars) | |