File size: 1,582 Bytes
f9bc0c1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
"""Conservative cleanup for Devanagari converter whitespace artifacts."""

from __future__ import annotations

import unicodedata


DEVANAGARI_VIRAMA = "\u094d"
DEVANAGARI_CONSONANT_RANGES = (
    ("\u0915", "\u0939"),
    ("\u0958", "\u095f"),
    ("\u0978", "\u097f"),
)


def _is_devanagari_consonant(char: str) -> bool:
    return any(start <= char <= end for start, end in DEVANAGARI_CONSONANT_RANGES)


def normalize_devanagari_whitespace(text: str) -> str:
    """Remove only known converter-artifact runs of literal spaces.

    A space run is removed when it is either directly between a Devanagari
    virama and consonant, or directly before any Unicode nonspacing mark.
    All other characters and whitespace are preserved verbatim.
    """

    normalized: list[str] = []
    index = 0
    while index < len(text):
        if text[index] != " ":
            normalized.append(text[index])
            index += 1
            continue

        run_end = index + 1
        while run_end < len(text) and text[run_end] == " ":
            run_end += 1

        previous = text[index - 1] if index else ""
        following = text[run_end] if run_end < len(text) else ""
        after_virama_before_consonant = previous == DEVANAGARI_VIRAMA and bool(following) and _is_devanagari_consonant(following)
        before_combining_mark = bool(following) and unicodedata.category(following) == "Mn"

        if not (after_virama_before_consonant or before_combining_mark):
            normalized.append(text[index:run_end])
        index = run_end

    return "".join(normalized)