Hugging Face's logo

Spaces:
vikaskajla26
/
vagdhenu
Paused

App Files Community
vagdhenu
File size: 5,521 Bytes
68f10e1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
#!/usr/bin/env python3
"""Normalize Devanagari text for Sanskrit TTS.

Operates on saṃhitā Devanagari from the Anuvyakhyana TTS metadata CSVs.
Output is Devanagari (+ Vedic extension chars ᳵ ᳶ ँ) ready for SLP1 G2P.

Rules:
1. Strip editorial bracketing: ( ) " " ' ' - and ZWNJ. Inner text is kept (all recited).
2. Strip verse-number blocks (e.g., trailing " १॥") and stray Devanagari/ASCII numerals.
3. Expand ॐ → ओम्.
4. Visarga before unvoiced k-varga (क ख) → jihvāmūlīya ᳵ.
   Visarga before unvoiced p-varga (प फ) → upadhmānīya ᳶ.
5. Anusvāra before a varga consonant → homorganic nasal + halant.
   Anusvāra before य → chandrabindu ँ (G2P will render as nasalized-y).
   Anusvāra before {र ल व श ष स ह} → keep as anusvāra (nasalized continuant).
   Anusvāra at word-end / before । ॥ → म् (proper /m/).
"""

import re

KVARGA_UNVOICED = 'कख'
PVARGA_UNVOICED = 'पफ'
KVARGA = 'कखगघङ'
CVARGA = 'चछजझञ'
TVARGA_RETRO = 'टठडढण'
TVARGA_DENT = 'तथदधन'
PVARGA = 'पफबभम'

VARGA_TO_NASAL = {}
for c in KVARGA: VARGA_TO_NASAL[c] = 'ङ्'
for c in CVARGA: VARGA_TO_NASAL[c] = 'ञ्'
for c in TVARGA_RETRO: VARGA_TO_NASAL[c] = 'ण्'
for c in TVARGA_DENT: VARGA_TO_NASAL[c] = 'न्'
for c in PVARGA: VARGA_TO_NASAL[c] = 'म्'
VARGA_ALL = set(KVARGA + CVARGA + TVARGA_RETRO + TVARGA_DENT + PVARGA)

NON_VARGA_KEEP = set('रलवशषसह')

JIHVAMULIYA = 'ᳵ'
UPADHMANIYA = 'ᳶ'
CHANDRABINDU = 'ँ'
VISARGA = 'ः'
ANUSVARA = 'ं'

STRIP_CHARS = set('()""\'\'“”‘’-‌')

WORD_END = set(' \t\n।॥')


def strip_editorial(text: str) -> str:
    return ''.join(c for c in text if c not in STRIP_CHARS)


def strip_numerals(text: str) -> str:
    # Verse-number block: optional ws + digits + optional ws + closing danda(s)
    text = re.sub(r'\s*[०-९0-9]+\s*[।॥]+', '॥', text)
    # Any stray digits
    text = re.sub(r'[०-९0-9]+', '', text)
    return text


def expand_om(text: str) -> str:
    return text.replace('ॐ', 'ओम्')


def rewrite_visarga(text: str) -> str:
    text = re.sub(f'{VISARGA}([{KVARGA_UNVOICED}])', JIHVAMULIYA + r'\1', text)
    text = re.sub(f'{VISARGA}([{PVARGA_UNVOICED}])', UPADHMANIYA + r'\1', text)
    return text


def rewrite_anusvara(text: str) -> str:
    out = []
    n = len(text)
    i = 0
    while i < n:
        ch = text[i]
        if ch == ANUSVARA:
            nxt = text[i+1] if i+1 < n else ''
            if nxt in VARGA_ALL:
                out.append(VARGA_TO_NASAL[nxt])
            elif nxt == 'य':
                out.append(CHANDRABINDU)
            elif nxt in NON_VARGA_KEEP:
                out.append(ANUSVARA)
            elif nxt == '' or nxt in WORD_END:
                out.append('म्')
            else:
                out.append(ANUSVARA)
        else:
            out.append(ch)
        i += 1
    return ''.join(out)


def collapse_ws(text: str) -> str:
    text = re.sub(r'\s+', ' ', text)
    # Collapse repeated danda groups left by verse-number stripping
    text = re.sub(r'(॥)(\s*॥)+', '॥', text)
    text = re.sub(r'(।)(\s*।)+', '।', text)
    return text.strip()


def normalize(text: str) -> str:
    text = strip_editorial(text)
    text = strip_numerals(text)
    text = expand_om(text)
    text = rewrite_visarga(text)
    text = rewrite_anusvara(text)
    text = collapse_ws(text)
    return text


if __name__ == '__main__':
    import argparse, csv, sys
    ap = argparse.ArgumentParser()
    ap.add_argument('--input', help='metadata CSV path; if omitted, run built-in samples')
    ap.add_argument('--limit', type=int, default=10)
    args = ap.parse_args()

    if not args.input:
        samples = [
            'ॐ॥ नारायणं निखिलपूर्णगुणैकदेहं निर्दोषमाप्यतममप्यखिलैः सुवाक्यैः। अस्योद्भवादिदमशेषविशेषतोऽपि वन्द्यं सदा प्रियतमं मम सन्नमामि॥ १॥',
            '(॥ ॐ अथातो ब्रह्मजिज्ञासा ॐ॥) ओतत्ववाची ह्योङ्कारः शास्त्रादौ॥ ९॥',
            '"द्रव्यं कर्म च कालश्च स्वभावो जीव एव च। यदनुग्रहतः सन्ति न सन्ति यदुपेक्षया॥" १३॥',
            'तमेव शास्त्रप्रभवं प्रणम्य जगद्गुरूणां गुरुमञ्जसैव। विशेषतो मे परमाख्यविद्याव्याख्यां करोम्यन्वपि चाहमेव॥ २॥',
            'संयोगः संस्कारः अंक पञ्च रङ्ग दुःख दुःप्रसह वाक्क',
        ]
        for s in samples:
            print('IN: ', s)
            print('OUT:', normalize(s))
            print()
        sys.exit(0)

    with open(args.input) as f:
        r = csv.DictReader(f, delimiter='|')
        for i, row in enumerate(r):
            if i >= args.limit: break
            t = row['text']
            print(f'#{row["shloka_no"]}')
            print('  IN: ', t)
            print('  OUT:', normalize(t))
            print()