Spaces:
Paused
Paused
| """ | |
| Single source of truth for every language SpeechM supports. | |
| Adding a language = adding one row to LANGUAGES below. The per-engine code | |
| maps, the Whisper prompt hints and the PDF font selection are all derived from | |
| it. Keep this in sync with frontend/src/lib/languages.ts. | |
| """ | |
| from typing import Dict, List, Optional | |
| # Script -> font file registered by the PDF exporter. | |
| # The files themselves are fetched in the Dockerfile; NotoSans-Regular covers | |
| # Latin, Cyrillic and Greek, so those three scripts share it. | |
| SCRIPT_FONTS: Dict[str, str] = { | |
| "latin": "NotoSans-Regular.ttf", | |
| "cyrillic": "NotoSans-Regular.ttf", | |
| "greek": "NotoSans-Regular.ttf", | |
| "devanagari": "NotoSansDevanagari-Regular.ttf", | |
| "bengali": "NotoSansBengali-Regular.ttf", | |
| "gurmukhi": "NotoSansGurmukhi-Regular.ttf", | |
| "gujarati": "NotoSansGujarati-Regular.ttf", | |
| "telugu": "NotoSansTelugu-Regular.ttf", | |
| "tamil": "NotoSansTamil-Regular.ttf", | |
| "kannada": "NotoSansKannada-Regular.ttf", | |
| "malayalam": "NotoSansMalayalam-Regular.ttf", | |
| "arabic": "NotoSansArabic-Regular.ttf", | |
| "hebrew": "NotoSansHebrew-Regular.ttf", | |
| "thai": "NotoSansThai-Regular.ttf", | |
| "hanSc": "NotoSansSC-Regular.ttf", | |
| "japanese": "NotoSansJP-Regular.ttf", | |
| "korean": "NotoSansKR-Regular.ttf", | |
| } | |
| class Language: | |
| """One supported language and its per-engine identifiers.""" | |
| __slots__ = ("code", "label", "nllb", "bcp47", "script", "sarvam", "indic", "prompt") | |
| def __init__( | |
| self, | |
| code: str, | |
| label: str, | |
| nllb: str, | |
| bcp47: str, | |
| script: str, | |
| sarvam: Optional[str] = None, | |
| indic: bool = False, | |
| prompt: str = "", | |
| ): | |
| self.code = code # ISO 639-1, what Whisper expects | |
| self.label = label | |
| self.nllb = nllb # FLORES-200, what NLLB expects | |
| self.bcp47 = bcp47 # Azure Speech / Web Speech API | |
| self.script = script | |
| self.sarvam = sarvam # Sarvam locale; None where Sarvam has no coverage | |
| self.indic = indic # eligible for IndicTrans2 routing against English | |
| self.prompt = prompt # Whisper initial_prompt hint | |
| LANGUAGES: List[Language] = [ | |
| # --- Indian languages --------------------------------------------------- | |
| Language("en", "English", "eng_Latn", "en-US", "latin", sarvam="en-IN"), | |
| Language("hi", "Hindi", "hin_Deva", "hi-IN", "devanagari", sarvam="hi-IN", indic=True, | |
| prompt="नमस्ते, क्या, क्यों, कैसे, धन्यवाद, भारत, समाचार, चर्चा।"), | |
| Language("te", "Telugu", "tel_Telu", "te-IN", "telugu", sarvam="te-IN", indic=True, | |
| prompt="వస్తున్నాయి, విదేశీ, మెరుపులు, ఆరంభంలో, విధ్వంసం, ధన్యవాదాలు, నమస్కారం, ప్రశ్నలు, చర్చ, భారత్."), | |
| Language("ta", "Tamil", "tam_Taml", "ta-IN", "tamil", sarvam="ta-IN", indic=True, | |
| prompt="வணக்கம், நன்றி."), | |
| Language("kn", "Kannada", "kan_Knda", "kn-IN", "kannada", sarvam="kn-IN", indic=True, | |
| prompt="ನಮಸ್ಕಾರ, ಧನ್ಯವಾದಗಳು."), | |
| Language("ml", "Malayalam", "mal_Mlym", "ml-IN", "malayalam", sarvam="ml-IN", indic=True, | |
| prompt="നമസ്കാരം, നന്ദി."), | |
| Language("bn", "Bengali", "ben_Beng", "bn-IN", "bengali", sarvam="bn-IN", indic=True), | |
| Language("mr", "Marathi", "mar_Deva", "mr-IN", "devanagari", sarvam="mr-IN", indic=True), | |
| Language("gu", "Gujarati", "guj_Gujr", "gu-IN", "gujarati", sarvam="gu-IN", indic=True), | |
| Language("pa", "Punjabi", "pan_Guru", "pa-IN", "gurmukhi", sarvam="pa-IN", indic=True), | |
| Language("ur", "Urdu", "urd_Arab", "ur-IN", "arabic"), | |
| # --- European ----------------------------------------------------------- | |
| Language("es", "Spanish", "spa_Latn", "es-ES", "latin"), | |
| Language("fr", "French", "fra_Latn", "fr-FR", "latin"), | |
| Language("de", "German", "deu_Latn", "de-DE", "latin"), | |
| Language("it", "Italian", "ita_Latn", "it-IT", "latin"), | |
| Language("pt", "Portuguese", "por_Latn", "pt-BR", "latin"), | |
| Language("nl", "Dutch", "nld_Latn", "nl-NL", "latin"), | |
| Language("pl", "Polish", "pol_Latn", "pl-PL", "latin"), | |
| Language("sv", "Swedish", "swe_Latn", "sv-SE", "latin"), | |
| Language("da", "Danish", "dan_Latn", "da-DK", "latin"), | |
| Language("nb", "Norwegian", "nob_Latn", "nb-NO", "latin"), | |
| Language("fi", "Finnish", "fin_Latn", "fi-FI", "latin"), | |
| Language("cs", "Czech", "ces_Latn", "cs-CZ", "latin"), | |
| Language("sk", "Slovak", "slk_Latn", "sk-SK", "latin"), | |
| Language("hu", "Hungarian", "hun_Latn", "hu-HU", "latin"), | |
| Language("ro", "Romanian", "ron_Latn", "ro-RO", "latin"), | |
| Language("hr", "Croatian", "hrv_Latn", "hr-HR", "latin"), | |
| Language("ca", "Catalan", "cat_Latn", "ca-ES", "latin"), | |
| Language("tr", "Turkish", "tur_Latn", "tr-TR", "latin"), | |
| Language("el", "Greek", "ell_Grek", "el-GR", "greek"), | |
| Language("ru", "Russian", "rus_Cyrl", "ru-RU", "cyrillic"), | |
| Language("uk", "Ukrainian", "ukr_Cyrl", "uk-UA", "cyrillic"), | |
| Language("bg", "Bulgarian", "bul_Cyrl", "bg-BG", "cyrillic"), | |
| # --- Middle East -------------------------------------------------------- | |
| Language("ar", "Arabic", "arb_Arab", "ar-SA", "arabic"), | |
| Language("he", "Hebrew", "heb_Hebr", "he-IL", "hebrew"), | |
| Language("fa", "Persian", "pes_Arab", "fa-IR", "arabic"), | |
| # --- East & Southeast Asia --------------------------------------------- | |
| Language("zh", "Chinese (Mandarin)", "zho_Hans", "zh-CN", "hanSc"), | |
| Language("ja", "Japanese", "jpn_Jpan", "ja-JP", "japanese"), | |
| Language("ko", "Korean", "kor_Hang", "ko-KR", "korean"), | |
| Language("vi", "Vietnamese", "vie_Latn", "vi-VN", "latin"), | |
| Language("th", "Thai", "tha_Thai", "th-TH", "thai"), | |
| Language("id", "Indonesian", "ind_Latn", "id-ID", "latin"), | |
| Language("ms", "Malay", "zsm_Latn", "ms-MY", "latin"), | |
| Language("tl", "Filipino", "tgl_Latn", "fil-PH", "latin"), | |
| # --- Africa ------------------------------------------------------------- | |
| Language("sw", "Swahili", "swh_Latn", "sw-KE", "latin"), | |
| Language("af", "Afrikaans", "afr_Latn", "af-ZA", "latin"), | |
| ] | |
| BY_CODE: Dict[str, Language] = {lang.code: lang for lang in LANGUAGES} | |
| BY_NLLB: Dict[str, Language] = {lang.nllb: lang for lang in LANGUAGES} | |
| def resolve(code: Optional[str]) -> Optional[Language]: | |
| """Look a language up by short code ('te') or FLORES-200 code ('tel_Telu').""" | |
| if not code: | |
| return None | |
| return BY_CODE.get(code) or BY_NLLB.get(code) | |
| def to_short(code: Optional[str], default: Optional[str] = None) -> Optional[str]: | |
| """Normalise any supported spelling to our canonical ISO 639-1 code.""" | |
| lang = resolve(code) | |
| return lang.code if lang else (default if default is not None else code) | |
| def to_nllb(code: Optional[str], default: str = "eng_Latn") -> str: | |
| """Normalise any supported spelling to a FLORES-200 code.""" | |
| lang = resolve(code) | |
| return lang.nllb if lang else (code or default) | |
| def to_bcp47(code: Optional[str], default: str = "en-US") -> str: | |
| """Azure Speech / Web Speech locale for a language.""" | |
| lang = resolve(code) | |
| return lang.bcp47 if lang else (code or default) | |
| def to_sarvam(code: Optional[str], default: str = "en-IN") -> str: | |
| """Sarvam locale. Falls back to the default where Sarvam has no coverage.""" | |
| lang = resolve(code) | |
| return lang.sarvam if lang and lang.sarvam else default | |
| def supports_sarvam(code: Optional[str]) -> bool: | |
| lang = resolve(code) | |
| return bool(lang and lang.sarvam) | |
| # Scripts whose glyphs need HarfBuzz shaping (reordering, conjuncts, cursive | |
| # joining) to render correctly. Latin, Cyrillic, Greek and CJK do not. | |
| SHAPING_SCRIPTS = { | |
| "devanagari", "bengali", "gurmukhi", "gujarati", | |
| "telugu", "tamil", "kannada", "malayalam", | |
| "arabic", "hebrew", "thai", | |
| } | |
| def script_for(code: Optional[str]) -> str: | |
| """Script a language is written in. | |
| Accepts short codes ("te"), FLORES-200 codes ("tel_Telu") and the legacy | |
| "<iso639-1>_<Script>" spellings ("te_Telu") that older clients still send. | |
| """ | |
| lang = resolve(code) | |
| if lang: | |
| return lang.script | |
| # Unknown code: match on the script suffix so old saved transcripts keep | |
| # rendering in the right script rather than silently falling back to Latin. | |
| if code and "_" in code: | |
| suffix = "_" + code.split("_", 1)[1] | |
| for candidate in LANGUAGES: | |
| if candidate.nllb.endswith(suffix): | |
| return candidate.script | |
| return "latin" | |
| def needs_shaping(code: Optional[str]) -> bool: | |
| """Whether the PDF exporter should turn text shaping on for this language.""" | |
| return script_for(code) in SHAPING_SCRIPTS | |
| def font_for(code: Optional[str]) -> str: | |
| """Font file able to render this language's script in an exported PDF.""" | |
| return SCRIPT_FONTS.get(script_for(code), SCRIPT_FONTS["latin"]) | |
| # Whisper initial_prompt hints, only where a script needs pinning. | |
| LANGUAGE_PROMPTS: Dict[str, str] = {lang.code: lang.prompt for lang in LANGUAGES if lang.prompt} | |
| # Languages IndicTrans2 handles better than NLLB when paired with English. | |
| INDIC_LANGS = {lang.code for lang in LANGUAGES if lang.indic} | |
| # Every font file the PDF exporter may ask for, for Dockerfile/preflight checks. | |
| REQUIRED_FONTS = sorted(set(SCRIPT_FONTS.values())) | |