Download ocr-tech/src/ocrtech/devanagari_whitespace.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 1.58 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/src/ocrtech/devanagari_whitespace.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/src/ocrtech/devanagari_whitespace.py
-
curl -L -o devanagari_whitespace.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/src/ocrtech/devanagari_whitespace.py
1.58 kB
| """Conservative cleanup for Devanagari converter whitespace artifacts.""" | |
| from __future__ import annotations | |
| import unicodedata | |
| DEVANAGARI_VIRAMA = "\u094d" | |
| DEVANAGARI_CONSONANT_RANGES = ( | |
| ("\u0915", "\u0939"), | |
| ("\u0958", "\u095f"), | |
| ("\u0978", "\u097f"), | |
| ) | |
| def _is_devanagari_consonant(char: str) -> bool: | |
| return any(start <= char <= end for start, end in DEVANAGARI_CONSONANT_RANGES) | |
| def normalize_devanagari_whitespace(text: str) -> str: | |
| """Remove only known converter-artifact runs of literal spaces. | |
| A space run is removed when it is either directly between a Devanagari | |
| virama and consonant, or directly before any Unicode nonspacing mark. | |
| All other characters and whitespace are preserved verbatim. | |
| """ | |
| normalized: list[str] = [] | |
| index = 0 | |
| while index < len(text): | |
| if text[index] != " ": | |
| normalized.append(text[index]) | |
| index += 1 | |
| continue | |
| run_end = index + 1 | |
| while run_end < len(text) and text[run_end] == " ": | |
| run_end += 1 | |
| previous = text[index - 1] if index else "" | |
| following = text[run_end] if run_end < len(text) else "" | |
| after_virama_before_consonant = previous == DEVANAGARI_VIRAMA and bool(following) and _is_devanagari_consonant(following) | |
| before_combining_mark = bool(following) and unicodedata.category(following) == "Mn" | |
| if not (after_virama_before_consonant or before_combining_mark): | |
| normalized.append(text[index:run_end]) | |
| index = run_end | |
| return "".join(normalized) | |