Update matn_sozlovchi.py

matn_sozlovchi.py

@@ -0,0 +1,268 @@
+"""
+O'zbek TTS uchun metadata.csv matnini normalizatsiya qiluvchi skript.
+Vocabulary da yo'q belgilarni avtomatik aniqlaydi va almashtiradi.
+"""
+
+import os
+import re
+import shutil
+from collections import defaultdict
+
+# ─────────────────────────────────────────
+# CONFIG — o'zingizga moslashtiring
+# ─────────────────────────────────────────
+DATASET_PATH = "/content/drive/MyDrive/tts/dataset_final"
+METADATA_FILE = "metadata.csv"
+SEPARATOR = "|"   # metadata.csv da ustunlar ajratuvchisi
+
+# ─────────────────────────────────────────
+# VOCABULARY — config dagi characters bilan mos
+# ─────────────────────────────────────────
+ALLOWED_CHARS = set(
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+    "O'o'G'g'ShshChch'"
+    "0123456789"
+    "!,.? "          # punctuations
+    "\n"
+)
+
+# ─────────────────────────────────────────
+# NORMALIZATSIYA JADVALI
+# Har bir "noto'g'ri" belgi → o'zbek ekvivalenti
+# ─────────────────────────────────────────
+REPLACEMENTS = {
+    # ── Apostrof turlari → oddiy apostrof ──
+    "\u2018": "'",   # '  (left single quotation)
+    "\u2019": "'",   # '  (right single quotation)
+    "\u02bc": "'",   # ʼ  (modifier letter apostrophe)
+    "\u02b9": "'",   # ʹ  (modifier letter prime)
+    "\u0060": "'",   # `  (grave accent)
+    "\u00b4": "'",   # ´  (acute accent)
+    "\u2032": "'",   # ′  (prime)
+
+    # ── Qo'shtirnoq → o'chirish ──
+    "\u201c": "",    # "  (left double quotation)
+    "\u201d": "",    # "  (right double quotation)
+    "\u00ab": "",    # «
+    "\u00bb": "",    # »
+    '"': "",
+
+    # ── Kirill harflari → lotin ──
+    "а": "a", "б": "b", "в": "v", "г": "g", "д": "d",
+    "е": "e", "ё": "yo", "ж": "j", "з": "z", "и": "i",
+    "й": "y", "к": "k", "л": "l", "м": "m", "н": "n",
+    "о": "o", "п": "p", "р": "r", "с": "s", "т": "t",
+    "у": "u", "ф": "f", "х": "x", "ц": "ts", "ч": "ch",
+    "ш": "sh", "щ": "sh", "ъ": "'", "ы": "i", "ь": "",
+    "э": "e", "ю": "yu", "я": "ya",
+    "А": "A", "Б": "B", "В": "V", "Г": "G", "Д": "D",
+    "Е": "E", "Ё": "Yo", "Ж": "J", "З": "Z", "И": "I",
+    "Й": "Y", "К": "K", "Л": "L", "М": "M", "Н": "N",
+    "О": "O", "П": "P", "Р": "R", "С": "S", "Т": "T",
+    "У": "U", "Ф": "F", "Х": "X", "Ц": "Ts", "Ч": "Ch",
+    "Ш": "Sh", "Щ": "Sh", "Ъ": "'", "Ы": "I", "Ь": "",
+    "Э": "E", "Ю": "Yu", "Я": "Ya",
+    # O'zbek kirill maxsus harflari
+    "ў": "o'", "қ": "q", "ғ": "g'", "ҳ": "h", "ң": "ng",
+    "Ў": "O'", "Қ": "Q", "Ғ": "G'", "Ҳ": "H", "Ң": "Ng",
+
+    # ── Maxsus lotin harflari → o'zbek ──
+    "á": "a", "à": "a", "â": "a", "ä": "a", "ā": "a",
+    "é": "e", "è": "e", "ê": "e", "ë": "e", "ě": "e",
+    "í": "i", "ì": "i", "î": "i", "ï": "i", "ı": "i",
+    "ó": "o", "ò": "o", "ô": "o", "ö": "o", "ő": "o",
+    "ú": "u", "ù": "u", "û": "u", "ü": "u", "ű": "u",
+    "ñ": "n", "ç": "ch", "ž": "zh", "š": "sh", "č": "ch",
+    "ř": "r", "ý": "y", "ķ": "k", "ļ": "l", "ņ": "n",
+    "ģ": "g", "ħ": "h", "ĵ": "j", "ŵ": "w",
+    "Á": "A", "É": "E", "Í": "I", "Ó": "O", "Ú": "U",
+    "Ñ": "N", "Ç": "Ch", "Ž": "Zh", "Š": "Sh", "Č": "Ch",
+
+    # ── Raqamli/maxsus belgilar → matn ──
+    "%": " foiz",
+    "°": " daraja",
+    "№": "raqam",
+    "§": "",
+    "©": "",
+    "®": "",
+    "™": "",
+    "…": "...",      # ellipsis → uch nuqta (keyin nuqta qoladi)
+    "–": "-",        # en dash → oddiy chiziq
+    "—": "-",        # em dash → oddiy chiziq
+    "\u00ad": "",    # soft hyphen → o'chirish
+    "\u200b": "",    # zero-width space
+    "\u200c": "",    # zero-width non-joiner
+    "\u200d": "",    # zero-width joiner
+    "\u00a0": " ",   # non-breaking space → oddiy bo'shliq
+    "\t": " ",       # tab → bo'shliq
+
+    # ── Brackets/slash → o'chirish ──
+    "(": "",
+    ")": "",
+    "[": "",
+    "]": "",
+    "{": "",
+    "}": "",
+    "/": " ",
+    "\\": " ",
+    "|": " ",        # separator bilan aralashmasligi uchun
+    "_": " ",
+    "@": " at ",
+    "#": "",
+    "$": "",
+    "^": "",
+    "*": "",
+    "+": " va ",
+    "=": " teng ",
+    "<": "",
+    ">": "",
+    "~": "",
+    "`": "'",
+    "&": " va ",
+    ";": ",",
+    ":": ",",
+}
+
+# ─────────────────────────────────────────
+# ASOSIY FUNKSIYALAR
+# ─────────────────────���───────────────────
+
+def normalize_text(text: str) -> str:
+    """Matnni normalizatsiya qiladi."""
+    # 1. Almashtirishlar
+    for bad_char, good_char in REPLACEMENTS.items():
+        text = text.replace(bad_char, good_char)
+
+    # 2. Raqamlarni so'z ko'rinishiga o'tkazish (oddiy)
+    # (Murakkabroq holatlar uchun num2words kutubxonasi kerak)
+
+    # 3. Ko'p bo'shliqlarni birlari qilish
+    text = re.sub(r' +', ' ', text)
+
+    # 4. Boshidagi va oxiridagi bo'shliqni tozalash
+    text = text.strip()
+
+    # 5. Hali ham ruxsat etilmagan belgilar qolsa — o'chirish
+    text = ''.join(c for c in text if c in ALLOWED_CHARS)
+
+    # 6. Yana ko'p bo'shliqlarni tozalash (o'chirishdan keyin)
+    text = re.sub(r' +', ' ', text).strip()
+
+    return text
+
+
+def analyze_metadata(filepath: str) -> dict:
+    """Metadata dagi noto'g'ri belgilarni hisoblab chiqaradi."""
+    bad_chars = defaultdict(int)
+    total_lines = 0
+    bad_lines = 0
+
+    with open(filepath, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            total_lines += 1
+            cols = line.split(SEPARATOR)
+            if len(cols) < 2:
+                continue
+            text = cols[1]
+            for char in text:
+                if char not in ALLOWED_CHARS:
+                    bad_chars[char] += 1
+                    bad_lines += 1
+
+    return {
+        "total": total_lines,
+        "bad_lines": bad_lines,
+        "bad_chars": dict(sorted(bad_chars.items(), key=lambda x: -x[1]))
+    }
+
+
+def process_metadata(dataset_path: str):
+    """Metadata ni o'qib, normalizatsiya qilib, qayta saqlaydi."""
+    filepath = os.path.join(dataset_path, METADATA_FILE)
+    backup_path = filepath + ".backup"
+
+    # ── Tahlil ──
+    print("=" * 55)
+    print("📊 TAHLIL — Noto'g'ri belgilar:")
+    stats = analyze_metadata(filepath)
+    print(f"   Jami qatorlar : {stats['total']}")
+    print(f"   Xato qatorlar : {stats['bad_lines']}")
+    print(f"\n   Belgi | Soni | Almashtiriladi")
+    print(f"   {'─'*40}")
+    for char, count in stats["bad_chars"].items():
+        replacement = REPLACEMENTS.get(char, "❌ O'CHIRILADI")
+        print(f"   '{char}' (U+{ord(char):04X}) | {count:4d} | → '{replacement}'")
+
+    if not stats["bad_chars"]:
+        print("   ✅ Hamma belgilar to'g'ri! Normalizatsiya shart emas.")
+        return
+
+    # ── Backup ──
+    print(f"\n💾 Backup saqlanmoqda: {backup_path}")
+    shutil.copy2(filepath, backup_path)
+
+    # ── Normalizatsiya ──
+    print(f"🔄 Normalizatsiya boshlanmoqda...")
+    fixed_lines = 0
+    skipped_lines = 0
+    output_lines = []
+
+    with open(filepath, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            cols = line.split(SEPARATOR)
+            if len(cols) < 2:
+                skipped_lines += 1
+                continue
+
+            wav_id = cols[0].strip()
+            text = cols[1].strip()
+            original_text = text
+
+            text = normalize_text(text)
+
+            # Bo'sh matn qolsa — o'tkazib yuboramiz
+            if not text:
+                skipped_lines += 1
+                continue
+
+            if text != original_text:
+                fixed_lines += 1
+
+            output_lines.append(f"{wav_id}{SEPARATOR}{text}")
+
+    # ── Saqlash ──
+    with open(filepath, "w", encoding="utf-8") as f:
+        f.write("\n".join(output_lines) + "\n")
+
+    # ── Natija ──
+    print(f"\n{'='*55}")
+    print(f"✅ NATIJA:")
+    print(f"   Tuzatilgan qatorlar : {fixed_lines}")
+    print(f"   O'tkazilgan (bo'sh) : {skipped_lines}")
+    print(f"   Saqlangan qatorlar  : {len(output_lines)}")
+    print(f"   Backup              : {backup_path}")
+
+    # ── Tekshiruv ──
+    print(f"\n🔍 Normalizatsiyadan keyin tekshiruv:")
+    stats_after = analyze_metadata(filepath)
+    if stats_after["bad_chars"]:
+        print(f"   ⚠️  Hali ham noto'g'ri belgilar bor:")
+        for char, count in stats_after["bad_chars"].items():
+            print(f"      '{char}' (U+{ord(char):04X}) → {count} marta")
+        print(f"\n   💡 Ularni REPLACEMENTS jadvaliga qo'shing va qayta ishga tushiring.")
+    else:
+        print(f"   ✅ Barcha belgilar to'g'ri! Metadata tayyor.")
+    print("=" * 55)
+
+
+# ─────────────────────────────────────────
+# ISHGA TUSHIRISH
+# ─────────────────────────────────────────
+if __name__ == "__main__":
+    process_metadata(DATASET_PATH)