""" Language Registry & Constants for AI4Bharat Pralekha Dataset Covers 12 Indic languages + English with full Unicode metadata. """ from __future__ import annotations from dataclasses import dataclass, field from typing import Dict, List, Optional, Tuple import unicodedata # ───────────────────────────────────────────── # Script Unicode Block Definitions # ───────────────────────────────────────────── SCRIPT_UNICODE_BLOCKS: Dict[str, List[Tuple[int, int]]] = { "Devanagari": [(0x0900, 0x097F), (0xA8E0, 0xA8FF)], "Bengali": [(0x0980, 0x09FF)], "Gurmukhi": [(0x0A00, 0x0A7F)], "Gujarati": [(0x0A80, 0x0AFF)], "Odia": [(0x0B00, 0x0B7F)], "Tamil": [(0x0B80, 0x0BFF)], "Telugu": [(0x0C00, 0x0C7F)], "Kannada": [(0x0C80, 0x0CFF)], "Malayalam": [(0x0D00, 0x0D7F)], "Sinhala": [(0x0D80, 0x0DFF)], "Arabic": [(0x0600, 0x06FF), (0x0750, 0x077F), (0xFB50, 0xFDFF), (0xFE70, 0xFEFF)], "Latin": [(0x0000, 0x007F), (0x0080, 0x00FF), (0x0100, 0x017F)], "Cyrillic": [(0x0400, 0x04FF)], "CJK": [(0x4E00, 0x9FFF), (0x3400, 0x4DBF)], } SCRIPT_TO_DIRECTION: Dict[str, str] = { "Devanagari": "LTR", "Bengali": "LTR", "Gurmukhi": "LTR", "Gujarati": "LTR", "Odia": "LTR", "Tamil": "LTR", "Telugu": "LTR", "Kannada": "LTR", "Malayalam": "LTR", "Arabic": "RTL", "Latin": "LTR", } @dataclass class LanguageInfo: code: str # Pralekha subset code (ben, hin, ...) name: str # Full English name script: str # Primary script name iso639_1: Optional[str] # ISO 639-1 two-letter code iso639_3: str # ISO 639-3 three-letter code label_id: int # Numeric label for ML models unicode_ranges: List[Tuple[int, int]] = field(default_factory=list) direction: str = "LTR" family: str = "Indo-Aryan" native_name: str = "" romanization_scheme: Optional[str] = None # IAST, ISO 15919, etc. shares_script_with: List[str] = field(default_factory=list) def char_in_script(self, char: str) -> bool: cp = ord(char) return any(lo <= cp <= hi for lo, hi in self.unicode_ranges) # ───────────────────────────────────────────── # Language Registry (12 Indic + English) # ───────────────────────────────────────────── LANGUAGE_REGISTRY: Dict[str, LanguageInfo] = { "ben": LanguageInfo( code="ben", name="Bengali", script="Bengali", iso639_1="bn", iso639_3="ben", label_id=0, unicode_ranges=SCRIPT_UNICODE_BLOCKS["Bengali"], direction="LTR", family="Indo-Aryan", native_name="বাংলা", romanization_scheme="ISO 15919", ), "eng": LanguageInfo( code="eng", name="English", script="Latin", iso639_1="en", iso639_3="eng", label_id=1, unicode_ranges=SCRIPT_UNICODE_BLOCKS["Latin"], direction="LTR", family="Germanic", native_name="English", ), "guj": LanguageInfo( code="guj", name="Gujarati", script="Gujarati", iso639_1="gu", iso639_3="guj", label_id=2, unicode_ranges=SCRIPT_UNICODE_BLOCKS["Gujarati"], direction="LTR", family="Indo-Aryan", native_name="ગુજરાતી", romanization_scheme="ISO 15919", ), "hin": LanguageInfo( code="hin", name="Hindi", script="Devanagari", iso639_1="hi", iso639_3="hin", label_id=3, unicode_ranges=SCRIPT_UNICODE_BLOCKS["Devanagari"], direction="LTR", family="Indo-Aryan", native_name="हिन्दी", romanization_scheme="IAST", shares_script_with=["mar"], ), "kan": LanguageInfo( code="kan", name="Kannada", script="Kannada", iso639_1="kn", iso639_3="kan", label_id=4, unicode_ranges=SCRIPT_UNICODE_BLOCKS["Kannada"], direction="LTR", family="Dravidian", native_name="ಕನ್ನಡ", ), "mal": LanguageInfo( code="mal", name="Malayalam", script="Malayalam", iso639_1="ml", iso639_3="mal", label_id=5, unicode_ranges=SCRIPT_UNICODE_BLOCKS["Malayalam"], direction="LTR", family="Dravidian", native_name="മലയാളം", ), "mar": LanguageInfo( code="mar", name="Marathi", script="Devanagari", iso639_1="mr", iso639_3="mar", label_id=6, unicode_ranges=SCRIPT_UNICODE_BLOCKS["Devanagari"], direction="LTR", family="Indo-Aryan", native_name="मराठी", romanization_scheme="IAST", shares_script_with=["hin"], ), "ori": LanguageInfo( code="ori", name="Odia", script="Odia", iso639_1="or", iso639_3="ori", label_id=7, unicode_ranges=SCRIPT_UNICODE_BLOCKS["Odia"], direction="LTR", family="Indo-Aryan", native_name="ଓଡ଼ିଆ", ), "pan": LanguageInfo( code="pan", name="Punjabi", script="Gurmukhi", iso639_1="pa", iso639_3="pan", label_id=8, unicode_ranges=SCRIPT_UNICODE_BLOCKS["Gurmukhi"], direction="LTR", family="Indo-Aryan", native_name="ਪੰਜਾਬੀ", ), "tam": LanguageInfo( code="tam", name="Tamil", script="Tamil", iso639_1="ta", iso639_3="tam", label_id=9, unicode_ranges=SCRIPT_UNICODE_BLOCKS["Tamil"], direction="LTR", family="Dravidian", native_name="தமிழ்", ), "tel": LanguageInfo( code="tel", name="Telugu", script="Telugu", iso639_1="te", iso639_3="tel", label_id=10, unicode_ranges=SCRIPT_UNICODE_BLOCKS["Telugu"], direction="LTR", family="Dravidian", native_name="తెలుగు", ), "urd": LanguageInfo( code="urd", name="Urdu", script="Arabic", iso639_1="ur", iso639_3="urd", label_id=11, unicode_ranges=SCRIPT_UNICODE_BLOCKS["Arabic"], direction="RTL", family="Indo-Aryan", native_name="اردو", romanization_scheme="ALA-LC", ), } # Derived mappings LABEL_TO_CODE: Dict[int, str] = {v.label_id: k for k, v in LANGUAGE_REGISTRY.items()} CODE_TO_LABEL: Dict[str, int] = {k: v.label_id for k, v in LANGUAGE_REGISTRY.items()} ALL_LANGUAGE_CODES: List[str] = sorted(LANGUAGE_REGISTRY.keys()) NUM_CLASSES: int = len(LANGUAGE_REGISTRY) # Script families for analysis DRAVIDIAN_LANGUAGES = {"kan", "mal", "tam", "tel"} INDO_ARYAN_LANGUAGES = {"ben", "guj", "hin", "mar", "ori", "pan", "urd"} DEVANAGARI_LANGUAGES = {"hin", "mar"} RTL_LANGUAGES = {"urd"} # Confusable pairs (share script or similar features) CONFUSABLE_PAIRS = [ ("hin", "mar"), # Both Devanagari ("tam", "mal"), # Related Dravidian scripts ("kan", "tel"), # Related Dravidian scripts ("hin", "urd"), # Same spoken language, different scripts ("ben", "ori"), # Visually similar scripts ]