File size: 3,019 Bytes
f9bc0c1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | """Unicode normalization and dictionary coverage helpers."""
from __future__ import annotations
import string
import unicodedata
DEVANAGARI_START = "\u0900"
DEVANAGARI_END = "\u097f"
DEVANAGARI_EXTENDED_START = "\ua8e0"
DEVANAGARI_EXTENDED_END = "\ua8ff"
PRESERVED_FORMAT_CHARS = {"\u200c", "\u200d"}
ASCII_PRINTABLE = set(string.printable)
def is_devanagari(char: str) -> bool:
return DEVANAGARI_START <= char <= DEVANAGARI_END or DEVANAGARI_EXTENDED_START <= char <= DEVANAGARI_EXTENDED_END
def normalize_ocr_text(text: str, *, collapse_spaces: bool = False) -> str:
"""Normalize OCR text while preserving Devanagari combining marks."""
normalized = unicodedata.normalize("NFC", text.replace("\u00a0", " "))
kept: list[str] = []
previous_space = False
for char in normalized:
category = unicodedata.category(char)
if category in {"Cc", "Cf"} and char not in {"\n", "\t", "\r"} | PRESERVED_FORMAT_CHARS:
continue
if collapse_spaces and char in {" ", "\t"}:
if previous_space:
continue
char = " "
previous_space = True
else:
previous_space = char == " "
kept.append(char)
return "".join(kept)
def script_counts(text: str) -> dict[str, int]:
counts = {"devanagari": 0, "latin": 0, "digit": 0, "punctuation": 0, "other": 0}
for char in normalize_ocr_text(text):
if char.isspace():
continue
if is_devanagari(char):
counts["devanagari"] += 1
elif "LATIN" in unicodedata.name(char, ""):
counts["latin"] += 1
elif char.isdigit():
counts["digit"] += 1
elif unicodedata.category(char).startswith("P") or char in string.punctuation:
counts["punctuation"] += 1
else:
counts["other"] += 1
return counts
def default_allowed_characters() -> set[str]:
chars = set(ASCII_PRINTABLE)
for codepoint in range(ord(DEVANAGARI_START), ord(DEVANAGARI_END) + 1):
chars.add(chr(codepoint))
for codepoint in range(ord(DEVANAGARI_EXTENDED_START), ord(DEVANAGARI_EXTENDED_END) + 1):
chars.add(chr(codepoint))
chars.update(PRESERVED_FORMAT_CHARS)
chars.update({"।", "॥", "–", "—", "‘", "’", "“", "”", "…", "₹", "₨"})
return chars
def unsupported_characters(text: str, allowed: set[str] | None = None) -> list[str]:
allowed_chars = allowed or default_allowed_characters()
unsupported = sorted({char for char in normalize_ocr_text(text) if char not in allowed_chars and not char.isspace()})
return unsupported
def dictionary_from_texts(texts: list[str], *, include_ascii: bool = True) -> list[str]:
chars: set[str] = set()
if include_ascii:
chars.update(ch for ch in string.printable if ch not in {"\x0b", "\x0c", "\r"})
for text in texts:
chars.update(char for char in normalize_ocr_text(text) if char not in {"\n", "\r", "\t"})
return sorted(chars)
|