""" Single source of truth for every language SpeechM supports. Adding a language = adding one row to LANGUAGES below. The per-engine code maps, the Whisper prompt hints and the PDF font selection are all derived from it. Keep this in sync with frontend/src/lib/languages.ts. """ from typing import Dict, List, Optional # Script -> font file registered by the PDF exporter. # The files themselves are fetched in the Dockerfile; NotoSans-Regular covers # Latin, Cyrillic and Greek, so those three scripts share it. SCRIPT_FONTS: Dict[str, str] = { "latin": "NotoSans-Regular.ttf", "cyrillic": "NotoSans-Regular.ttf", "greek": "NotoSans-Regular.ttf", "devanagari": "NotoSansDevanagari-Regular.ttf", "bengali": "NotoSansBengali-Regular.ttf", "gurmukhi": "NotoSansGurmukhi-Regular.ttf", "gujarati": "NotoSansGujarati-Regular.ttf", "telugu": "NotoSansTelugu-Regular.ttf", "tamil": "NotoSansTamil-Regular.ttf", "kannada": "NotoSansKannada-Regular.ttf", "malayalam": "NotoSansMalayalam-Regular.ttf", "arabic": "NotoSansArabic-Regular.ttf", "hebrew": "NotoSansHebrew-Regular.ttf", "thai": "NotoSansThai-Regular.ttf", "hanSc": "NotoSansSC-Regular.ttf", "japanese": "NotoSansJP-Regular.ttf", "korean": "NotoSansKR-Regular.ttf", } class Language: """One supported language and its per-engine identifiers.""" __slots__ = ("code", "label", "nllb", "bcp47", "script", "sarvam", "indic", "prompt") def __init__( self, code: str, label: str, nllb: str, bcp47: str, script: str, sarvam: Optional[str] = None, indic: bool = False, prompt: str = "", ): self.code = code # ISO 639-1, what Whisper expects self.label = label self.nllb = nllb # FLORES-200, what NLLB expects self.bcp47 = bcp47 # Azure Speech / Web Speech API self.script = script self.sarvam = sarvam # Sarvam locale; None where Sarvam has no coverage self.indic = indic # eligible for IndicTrans2 routing against English self.prompt = prompt # Whisper initial_prompt hint LANGUAGES: List[Language] = [ # --- Indian languages --------------------------------------------------- Language("en", "English", "eng_Latn", "en-US", "latin", sarvam="en-IN"), Language("hi", "Hindi", "hin_Deva", "hi-IN", "devanagari", sarvam="hi-IN", indic=True, prompt="नमस्ते, क्या, क्यों, कैसे, धन्यवाद, भारत, समाचार, चर्चा।"), Language("te", "Telugu", "tel_Telu", "te-IN", "telugu", sarvam="te-IN", indic=True, prompt="వస్తున్నాయి, విదేశీ, మెరుపులు, ఆరంభంలో, విధ్వంసం, ధన్యవాదాలు, నమస్కారం, ప్రశ్నలు, చర్చ, భారత్."), Language("ta", "Tamil", "tam_Taml", "ta-IN", "tamil", sarvam="ta-IN", indic=True, prompt="வணக்கம், நன்றி."), Language("kn", "Kannada", "kan_Knda", "kn-IN", "kannada", sarvam="kn-IN", indic=True, prompt="ನಮಸ್ಕಾರ, ಧನ್ಯವಾದಗಳು."), Language("ml", "Malayalam", "mal_Mlym", "ml-IN", "malayalam", sarvam="ml-IN", indic=True, prompt="നമസ്കാരം, നന്ദി."), Language("bn", "Bengali", "ben_Beng", "bn-IN", "bengali", sarvam="bn-IN", indic=True), Language("mr", "Marathi", "mar_Deva", "mr-IN", "devanagari", sarvam="mr-IN", indic=True), Language("gu", "Gujarati", "guj_Gujr", "gu-IN", "gujarati", sarvam="gu-IN", indic=True), Language("pa", "Punjabi", "pan_Guru", "pa-IN", "gurmukhi", sarvam="pa-IN", indic=True), Language("ur", "Urdu", "urd_Arab", "ur-IN", "arabic"), # --- European ----------------------------------------------------------- Language("es", "Spanish", "spa_Latn", "es-ES", "latin"), Language("fr", "French", "fra_Latn", "fr-FR", "latin"), Language("de", "German", "deu_Latn", "de-DE", "latin"), Language("it", "Italian", "ita_Latn", "it-IT", "latin"), Language("pt", "Portuguese", "por_Latn", "pt-BR", "latin"), Language("nl", "Dutch", "nld_Latn", "nl-NL", "latin"), Language("pl", "Polish", "pol_Latn", "pl-PL", "latin"), Language("sv", "Swedish", "swe_Latn", "sv-SE", "latin"), Language("da", "Danish", "dan_Latn", "da-DK", "latin"), Language("nb", "Norwegian", "nob_Latn", "nb-NO", "latin"), Language("fi", "Finnish", "fin_Latn", "fi-FI", "latin"), Language("cs", "Czech", "ces_Latn", "cs-CZ", "latin"), Language("sk", "Slovak", "slk_Latn", "sk-SK", "latin"), Language("hu", "Hungarian", "hun_Latn", "hu-HU", "latin"), Language("ro", "Romanian", "ron_Latn", "ro-RO", "latin"), Language("hr", "Croatian", "hrv_Latn", "hr-HR", "latin"), Language("ca", "Catalan", "cat_Latn", "ca-ES", "latin"), Language("tr", "Turkish", "tur_Latn", "tr-TR", "latin"), Language("el", "Greek", "ell_Grek", "el-GR", "greek"), Language("ru", "Russian", "rus_Cyrl", "ru-RU", "cyrillic"), Language("uk", "Ukrainian", "ukr_Cyrl", "uk-UA", "cyrillic"), Language("bg", "Bulgarian", "bul_Cyrl", "bg-BG", "cyrillic"), # --- Middle East -------------------------------------------------------- Language("ar", "Arabic", "arb_Arab", "ar-SA", "arabic"), Language("he", "Hebrew", "heb_Hebr", "he-IL", "hebrew"), Language("fa", "Persian", "pes_Arab", "fa-IR", "arabic"), # --- East & Southeast Asia --------------------------------------------- Language("zh", "Chinese (Mandarin)", "zho_Hans", "zh-CN", "hanSc"), Language("ja", "Japanese", "jpn_Jpan", "ja-JP", "japanese"), Language("ko", "Korean", "kor_Hang", "ko-KR", "korean"), Language("vi", "Vietnamese", "vie_Latn", "vi-VN", "latin"), Language("th", "Thai", "tha_Thai", "th-TH", "thai"), Language("id", "Indonesian", "ind_Latn", "id-ID", "latin"), Language("ms", "Malay", "zsm_Latn", "ms-MY", "latin"), Language("tl", "Filipino", "tgl_Latn", "fil-PH", "latin"), # --- Africa ------------------------------------------------------------- Language("sw", "Swahili", "swh_Latn", "sw-KE", "latin"), Language("af", "Afrikaans", "afr_Latn", "af-ZA", "latin"), ] BY_CODE: Dict[str, Language] = {lang.code: lang for lang in LANGUAGES} BY_NLLB: Dict[str, Language] = {lang.nllb: lang for lang in LANGUAGES} def resolve(code: Optional[str]) -> Optional[Language]: """Look a language up by short code ('te') or FLORES-200 code ('tel_Telu').""" if not code: return None return BY_CODE.get(code) or BY_NLLB.get(code) def to_short(code: Optional[str], default: Optional[str] = None) -> Optional[str]: """Normalise any supported spelling to our canonical ISO 639-1 code.""" lang = resolve(code) return lang.code if lang else (default if default is not None else code) def to_nllb(code: Optional[str], default: str = "eng_Latn") -> str: """Normalise any supported spelling to a FLORES-200 code.""" lang = resolve(code) return lang.nllb if lang else (code or default) def to_bcp47(code: Optional[str], default: str = "en-US") -> str: """Azure Speech / Web Speech locale for a language.""" lang = resolve(code) return lang.bcp47 if lang else (code or default) def to_sarvam(code: Optional[str], default: str = "en-IN") -> str: """Sarvam locale. Falls back to the default where Sarvam has no coverage.""" lang = resolve(code) return lang.sarvam if lang and lang.sarvam else default def supports_sarvam(code: Optional[str]) -> bool: lang = resolve(code) return bool(lang and lang.sarvam) # Scripts whose glyphs need HarfBuzz shaping (reordering, conjuncts, cursive # joining) to render correctly. Latin, Cyrillic, Greek and CJK do not. SHAPING_SCRIPTS = { "devanagari", "bengali", "gurmukhi", "gujarati", "telugu", "tamil", "kannada", "malayalam", "arabic", "hebrew", "thai", } def script_for(code: Optional[str]) -> str: """Script a language is written in. Accepts short codes ("te"), FLORES-200 codes ("tel_Telu") and the legacy "_