ocr-projects-archive / ocr-tech /src /ocrtech /devanagari_whitespace.py
prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
1.58 kB
"""Conservative cleanup for Devanagari converter whitespace artifacts."""
from __future__ import annotations
import unicodedata
DEVANAGARI_VIRAMA = "\u094d"
DEVANAGARI_CONSONANT_RANGES = (
("\u0915", "\u0939"),
("\u0958", "\u095f"),
("\u0978", "\u097f"),
)
def _is_devanagari_consonant(char: str) -> bool:
return any(start <= char <= end for start, end in DEVANAGARI_CONSONANT_RANGES)
def normalize_devanagari_whitespace(text: str) -> str:
"""Remove only known converter-artifact runs of literal spaces.
A space run is removed when it is either directly between a Devanagari
virama and consonant, or directly before any Unicode nonspacing mark.
All other characters and whitespace are preserved verbatim.
"""
normalized: list[str] = []
index = 0
while index < len(text):
if text[index] != " ":
normalized.append(text[index])
index += 1
continue
run_end = index + 1
while run_end < len(text) and text[run_end] == " ":
run_end += 1
previous = text[index - 1] if index else ""
following = text[run_end] if run_end < len(text) else ""
after_virama_before_consonant = previous == DEVANAGARI_VIRAMA and bool(following) and _is_devanagari_consonant(following)
before_combining_mark = bool(following) and unicodedata.category(following) == "Mn"
if not (after_virama_before_consonant or before_combining_mark):
normalized.append(text[index:run_end])
index = run_end
return "".join(normalized)