EchoScript : Fixes : Why Persian/Portuguese/Turkish were failing , Korean removed from Marian targets , Multiple clicks , UI improvements

app.py

@@ -90,8 +90,9 @@ SOURCE_LANGUAGE_CHOICES = ["Auto Detect"] + list(SUPPORTED_LANGUAGES.values())
 
 # Label -> ISO 639-1 code, built from the full (Anthropic) superset so it
 # resolves correctly regardless of which list is currently offered.
+# Labels are just language names (e.g. "French", not "French Translation").
 _TRANSLATION_LABEL_TO_CODE = {
-    f"{name} Translation": code for code, name in ANTHROPIC_TARGET_LANGUAGES.items()
+    name: code for code, name in ANTHROPIC_TARGET_LANGUAGES.items()
 }
 
 # Sections are pre-built for every language in the full superset (hidden by
@@ -101,21 +102,12 @@ _TRANSLATION_SECTION_ORDER = list(_TRANSLATION_LABEL_TO_CODE.keys())
 
 
 def _compute_translation_choices(has_key: bool, exclude_code: Optional[str]) -> list[str]:
-    """The translate-to picker, cascaded from key presence + source language.
-
-    `exclude_code` is the actual transcript language at this point (this
-    picker only exists once a Transcript does), never a pre-detection
-    guess. Without a key, the Marian options are checked for real against
-    the Hub for this specific source language -- not a fixed guess -- so
-    a source with no path to a given language (direct or English-pivot)
-    simply won't offer it, and one with good coverage offers everything
-    that's actually reachable (which can be more than four languages).
-    """
+    """The translate-to picker, cascaded from key presence + source language."""
     if has_key:
         pool = ANTHROPIC_TARGET_LANGUAGES
     else:
         pool = available_marian_targets(exclude_code) if exclude_code else {}
-    return [f"{name} Translation" for code, name in pool.items() if code != exclude_code]
+    return [name for code, name in pool.items() if code != exclude_code]
 
 
 def _on_api_key_change(api_key: str, cached_transcript: Optional[Transcript], current_value: list[str]):
@@ -287,9 +279,9 @@ def generate_translations(
 
     # ------------------------------------------------------------------
     # Pass 1: Resolve every section immediately from the cache or by
-    # hiding unselected ones. Only sections that need a real translation
-    # call are left as None (no-op) so their current UI state is
-    # preserved while we wait.
+    # hiding unselected ones. Sections that need a real translation show
+    # a "⏳ Translating..." placeholder so the user sees all boxes right
+    # away rather than having to wait for each one to appear.
     # ------------------------------------------------------------------
     section_states: dict[str, Optional[tuple]] = {}
     needs_translation: list[str] = []
@@ -303,11 +295,11 @@ def generate_translations(
             file_path = _write_text_file(text, tmp_dir, f"{code}_cached.txt")
             section_states[label] = (True, text, file_path)
         else:
-            # Will be computed in pass 2; leave as None for now.
-            section_states[label] = None
+            # Show the box immediately with a placeholder; fill it in pass 2.
+            section_states[label] = (True, "⏳ Translating...", None)
             needs_translation.append(label)
 
-    # Yield immediately so cached results appear without waiting for new ones.
+    # Yield immediately so cached/placeholder results appear at once.
     yield _make_outputs(section_states)
 
     # ------------------------------------------------------------------
@@ -365,7 +357,7 @@ with gr.Blocks(title="EchoScript") as demo:
 **Upload Audio → Select Audio Window → Detect Language & Generate Transcript
 → Preview & Choose Languages → Generate Translations → Copy / Download**
 
-<sub>build: 2026-07-01 04:55 UTC &middot; static pair table (Persian fixed) &middot; generator translations (no spinner on cached)</sub>
+<sub>build: 2026-07-02 01:21 UTC &middot; fixed fa/pt/tr model names &middot; short labels &middot; Select All</sub>
         """
     )
 
@@ -428,6 +420,9 @@ with gr.Blocks(title="EchoScript") as demo:
                     )
                     with gr.Group(visible=False) as translations_picker_group:
                         gr.Markdown("Translate to:")
+                        with gr.Row():
+                            select_all_btn = gr.Button("Select All", size="sm")
+                            select_none_btn = gr.Button("Deselect All", size="sm")
                         translate_choices_input = gr.CheckboxGroup(choices=[], value=[], label=None)
                         generate_translations_button = gr.Button("Generate Translations", variant="primary")
 
@@ -526,6 +521,16 @@ with gr.Blocks(title="EchoScript") as demo:
         outputs=[translate_choices_input],
     )
 
+    select_all_btn.click(
+        fn=lambda choices: gr.update(value=choices),
+        inputs=[translate_choices_input],
+        outputs=[translate_choices_input],
+    )
+    select_none_btn.click(
+        fn=lambda: gr.update(value=[]),
+        outputs=[translate_choices_input],
+    )
+
     reset_button.click(fn=reset_session_state, outputs=transcript_stage_outputs)
 
 if __name__ == "__main__":

services/translation.py

@@ -2,27 +2,26 @@
 
 Per the EchoScript v1.0 architecture decision, translation is ALWAYS
 derived from the canonical Transcript's text, never from the original
-audio:
-
-    Audio -> Transcript -> Translation   (allowed)
-    Audio -> Translation                 (never)
+audio.
 
 Two interchangeable backends, chosen per-request based on whether an
 Anthropic API key is supplied -- never stored:
 
-- "anthropic": the caller supplies their own API key. Sends transcript
-  text to Claude for translation. No missing-language-pair failure mode --
-  every supported language translates directly to every other one in a
-  single call. The key is passed straight through to the Anthropic client
-  for that one call and is never written to disk, logged, or cached.
-- "marian": fully offline, no API key needed. Uses local Helsinki-NLP
-  MarianMT models. Available targets per source language are determined
-  from a verified static table (see KNOWN_MARIAN_PAIRS below) rather
-  than live Hub API checks, which are unreliable in a rate-limited
-  unauthenticated HF Spaces environment.
-
-The UI is expected to only offer ANTHROPIC_TARGET_LANGUAGES once a key
-has been entered, and available_marian_targets(source) otherwise.
+- "anthropic": uses Claude for translation via the supplied key. No
+  missing-language-pair failure mode. Key is used only for this call.
+- "marian": fully offline, no API key needed. Uses Helsinki-NLP MarianMT
+  models. Available targets are determined from KNOWN_MARIAN_PAIRS below.
+
+IMPORTANT -- Helsinki-NLP model naming quirks addressed here:
+  - Some pairs use multilingual group models (e.g. "en-iir" for all
+    Indo-Iranian languages) rather than a direct bilingual model.
+  - Those multilingual models require a >>langcode<< token prepended to
+    every source segment so the model knows which target language to use.
+  - The repo suffix and the optional prefix token are stored together in
+    KNOWN_MARIAN_PAIRS so the translation code can apply them correctly.
+  - Japanese uses repo code "jap" not "ja" for the en->ja direction.
+  - Korean (en->ko) has no confirmed Helsinki-NLP model; it is omitted
+    from the Marian pair table so it only appears when an API key exists.
 """
 
 from __future__ import annotations
@@ -55,94 +54,89 @@ LANGUAGE_NAMES: dict[str, str] = {
     "ko": "Korean",
 }
 
-# Target languages offered once the person supplies their own Anthropic
-# API key -- Claude has no missing-pair problem so the list is simply
-# everything we know the name of.
 ANTHROPIC_TARGET_LANGUAGES: dict[str, str] = dict(LANGUAGE_NAMES)
 
 # ---------------------------------------------------------------------------
 # Verified Marian pair table
 #
-# Verified from the Helsinki-NLP Hub catalog. We use a static table rather
-# than live Hub API checks because:
-#   - The HF Hub returns 401 (Unauthorized) for nonexistent repos when
-#     requests are unauthenticated, and the huggingface_hub library raises
-#     that as RepositoryNotFoundError -- indistinguishable from a genuine
-#     404 without parsing the status code correctly.
-#   - Unauthenticated HF Spaces requests are aggressively rate-limited.
-#     A burst of ~30 simultaneous existence checks (14 candidate languages
-#     × 2-3 pivot steps each) reliably triggers 429s.
-#   - Both 401 and 429 were being silently swallowed into "not available",
-#     causing Persian to disappear from English's target list even though
-#     Helsinki-NLP/opus-mt-en-fa genuinely exists.
+# Format: (source_code, target_code) -> (repo_suffix, prefix_token)
+#
+# repo_suffix: appended to "Helsinki-NLP/opus-mt-" to form the model name.
+# prefix_token: prepended to every source segment for multilingual models
+#   that need a >>langcode<< token to select the target language, or None
+#   for standard bilingual models.
 #
-# Format: {(source_code, target_code): repo_suffix}
-# repo_suffix is what goes after "Helsinki-NLP/opus-mt-". In almost all
-# cases it's just f"{src}-{tgt}", but Helsinki-NLP uses "jap" instead of
-# the ISO "ja" for the en->Japanese model.
+# Verified against the Helsinki-NLP catalog. Key findings:
+#   - opus-mt-en-fa does NOT exist. Use opus-mt-en-iir (Indo-Iranian
+#     group) with >>pes<< (Western Persian, ISO 639-3).
+#   - opus-mt-en-pt does NOT exist. Use opus-mt-en-roa (Romance group)
+#     with >>por<< (Portuguese, ISO 639-3).
+#   - opus-mt-en-tr does NOT exist. Use opus-mt-en-trk (Turkic group)
+#     with >>tur<< (Turkish, ISO 639-3).
+#   - opus-mt-en-ko has no confirmed model; Korean removed from en->X.
+#   - opus-mt-en-jap (note: "jap" not "ja") for English->Japanese.
+#   - fa-en uses direct opus-mt-fa-en (not the group model).
+#   - pt-en uses opus-mt-roa-en (Romance group; pt-en not on Hub).
 # ---------------------------------------------------------------------------
-KNOWN_MARIAN_PAIRS: dict[tuple[str, str], str] = {
-    # English <-> everything
-    ("en", "fr"): "en-fr",
-    ("fr", "en"): "fr-en",
-    ("en", "de"): "en-de",
-    ("de", "en"): "de-en",
-    ("en", "fa"): "en-fa",   # Helsinki-NLP/opus-mt-en-fa -- confirmed via catalog
-    ("fa", "en"): "fa-en",   # Helsinki-NLP/opus-mt-fa-en -- confirmed earlier
-    ("en", "es"): "en-es",
-    ("es", "en"): "es-en",
-    ("en", "it"): "en-it",
-    ("it", "en"): "it-en",
-    ("en", "pt"): "en-pt",
-    ("pt", "en"): "pt-en",
-    ("en", "nl"): "en-nl",
-    ("nl", "en"): "nl-en",
-    ("en", "ar"): "en-ar",
-    ("ar", "en"): "ar-en",
-    ("en", "ru"): "en-ru",
-    ("ru", "en"): "ru-en",
-    ("en", "tr"): "en-tr",
-    ("tr", "en"): "tr-en",
-    ("en", "zh"): "en-zh",
-    ("zh", "en"): "zh-en",
-    ("en", "ko"): "en-ko",
-    ("ko", "en"): "ko-en",
-    ("en", "ja"): "en-jap",  # repo uses "jap" not "ja"
-    ("ja", "en"): "ja-en",
-    # Selected direct non-English pairs (common enough to avoid a pivot hop)
-    ("fr", "de"): "fr-de",
-    ("de", "fr"): "de-fr",
-    ("fr", "es"): "fr-es",
-    ("es", "fr"): "es-fr",
-    ("de", "es"): "de-es",
-    ("es", "de"): "es-de",
+KNOWN_MARIAN_PAIRS: dict[tuple[str, str], tuple[str, Optional[str]]] = {
+    # English -> X
+    ("en", "fr"): ("en-fr",  None),
+    ("en", "de"): ("en-de",  None),
+    ("en", "fa"): ("en-iir", ">>pes<<"),  # Indo-Iranian group, Western Persian
+    ("en", "es"): ("en-es",  None),
+    ("en", "it"): ("en-it",  None),
+    ("en", "pt"): ("en-roa", ">>por<<"),  # Romance group, Portuguese
+    ("en", "nl"): ("en-nl",  None),
+    ("en", "ar"): ("en-ar",  None),
+    ("en", "ru"): ("en-ru",  None),
+    ("en", "tr"): ("en-trk", ">>tur<<"),  # Turkic group, Turkish
+    ("en", "zh"): ("en-zh",  None),
+    ("en", "ja"): ("en-jap", None),       # Note: "jap" not "ja"
+    # en->ko omitted: no confirmed Helsinki-NLP model
+
+    # X -> English
+    ("fr", "en"): ("fr-en", None),
+    ("de", "en"): ("de-en", None),
+    ("fa", "en"): ("fa-en", None),
+    ("es", "en"): ("es-en", None),
+    ("it", "en"): ("it-en", None),
+    ("pt", "en"): ("roa-en", None),   # Romance group -> English
+    ("nl", "en"): ("nl-en", None),
+    ("ar", "en"): ("ar-en", None),
+    ("ru", "en"): ("ru-en", None),
+    ("tr", "en"): ("tr-en", None),
+    ("zh", "en"): ("zh-en", None),
+    ("ja", "en"): ("ja-en", None),
+    ("ko", "en"): ("ko-en", None),
+
+    # Selected direct non-English pairs (avoids double pivot hop)
+    ("fr", "de"): ("fr-de", None),
+    ("de", "fr"): ("de-fr", None),
+    ("fr", "es"): ("fr-es", None),
+    ("es", "fr"): ("es-fr", None),
+    ("de", "es"): ("de-es", None),
+    ("es", "de"): ("es-de", None),
 }
 
 
-def _marian_repo_name(src: str, tgt: str) -> Optional[str]:
-    """Return the Helsinki-NLP repo suffix for a pair, or None if unknown."""
+def _marian_entry(src: str, tgt: str) -> Optional[tuple[str, Optional[str]]]:
     return KNOWN_MARIAN_PAIRS.get((src, tgt))
 
 
-def _marian_direct_exists(src: str, tgt: str) -> bool:
-    return _marian_repo_name(src, tgt) is not None
-
-
-def _marian_path_repo_names(src: str, tgt: str) -> Optional[list[str]]:
-    """Return the list of repo suffixes needed to translate src->tgt.
+def _marian_path(src: str, tgt: str) -> Optional[list[tuple[str, Optional[str]]]]:
+    """Return the list of (repo_suffix, prefix_token) steps for src->tgt.
 
-    Returns a 1-element list for a direct pair, a 2-element list for an
-    English-pivot hop, or None if no path is known.
+    Returns a 1-step list for a direct (or multilingual-model) pair, a
+    2-step list for an English pivot, or None if no path is known.
     """
     if src == tgt:
         return None
-    direct = _marian_repo_name(src, tgt)
-    if direct:
-        return [direct]
-    # English pivot: src->en->tgt
+    entry = _marian_entry(src, tgt)
+    if entry:
+        return [entry]
     if src != "en" and tgt != "en":
-        hop1 = _marian_repo_name(src, "en")
-        hop2 = _marian_repo_name("en", tgt)
+        hop1 = _marian_entry(src, "en")
+        hop2 = _marian_entry("en", tgt)
         if hop1 and hop2:
             return [hop1, hop2]
     return None
@@ -151,13 +145,12 @@ def _marian_path_repo_names(src: str, tgt: str) -> Optional[list[str]]:
 def available_marian_targets(source_language: str) -> dict[str, str]:
     """Every language MarianMT can reach from `source_language`.
 
-    Based on the verified KNOWN_MARIAN_PAIRS table (direct pair or English
-    pivot). No network calls are made; the table is the source of truth.
+    Based on the verified KNOWN_MARIAN_PAIRS table. No network calls.
     """
     return {
         code: name
         for code, name in LANGUAGE_NAMES.items()
-        if code != source_language and _marian_path_repo_names(source_language, code) is not None
+        if code != source_language and _marian_path(source_language, code) is not None
     }
 
 
@@ -170,11 +163,12 @@ _ANTHROPIC_BATCH_SIZE = 40
 _NUMBERED_LINE_RE = re.compile(r"^\s*(\d+)[.\)]\s?(.*)$")
 
 
+class TranslationError(RuntimeError):
+    pass
+
+
 def _anthropic_client(api_key: str):
-    """Build a fresh Anthropic client for this one call. Deliberately not
-    cached: the key must not linger in module-level state."""
     import anthropic
-
     return anthropic.Anthropic(api_key=api_key)
 
 
@@ -184,7 +178,6 @@ def _translate_batch_via_anthropic(
     source_name = LANGUAGE_NAMES.get(source_language, source_language)
     target_name = LANGUAGE_NAMES.get(target_language, target_language)
     numbered_input = "\n".join(f"{i + 1}. {text}" for i, text in enumerate(texts))
-
     try:
         response = _anthropic_client(api_key).messages.create(
             model=_ANTHROPIC_MODEL,
@@ -193,50 +186,49 @@ def _translate_batch_via_anthropic(
                 f"You translate transcript lines from {source_name} to {target_name}. "
                 "You will be given a numbered list of lines, one sentence or fragment "
                 "per line. Reply with the same numbers, translated, one per line, in "
-                "the same order. Keep the same number of lines as the input -- never "
-                "merge, split, drop, or add lines. Output only the numbered translated "
-                "lines, with no preamble, no explanations, and no extra commentary."
+                "the same order. Keep the same number of lines as the input. "
+                "Output only the numbered translated lines, no preamble or commentary."
             ),
             messages=[{"role": "user", "content": numbered_input}],
         )
     except Exception as exc:
         raise TranslationError(
-            f"Anthropic translation request failed for "
-            f"'{source_language}' -> '{target_language}': {exc}"
+            f"Anthropic request failed for '{source_language}'->'{target_language}': {exc}"
         ) from exc
 
-    raw_text = "".join(
-        block.text for block in response.content if getattr(block, "type", None) == "text"
+    raw = "".join(
+        b.text for b in response.content if getattr(b, "type", None) == "text"
     )
     parsed: dict[int, str] = {}
-    for line in raw_text.splitlines():
-        match = _NUMBERED_LINE_RE.match(line)
-        if match:
-            parsed[int(match.group(1))] = match.group(2).strip()
+    for line in raw.splitlines():
+        m = _NUMBERED_LINE_RE.match(line)
+        if m:
+            parsed[int(m.group(1))] = m.group(2).strip()
 
     if len(parsed) != len(texts) or any((i + 1) not in parsed for i in range(len(texts))):
         raise TranslationError(
-            f"Anthropic response didn't match expected line count for "
-            f"'{source_language}' -> '{target_language}' "
-            f"(expected {len(texts)}, parsed {len(parsed)})."
+            f"Anthropic response line count mismatch for "
+            f"'{source_language}'->'{target_language}' "
+            f"(expected {len(texts)}, got {len(parsed)})."
         )
     return [parsed[i + 1] for i in range(len(texts))]
 
 
 def _translate_segments_via_anthropic(
-    segments: list[Segment], source_language: str, target_language: str, api_key: str
+    segments: list[Segment], src: str, tgt: str, api_key: str
 ) -> list[Segment]:
     non_empty = [(i, seg) for i, seg in enumerate(segments) if seg.text]
-    translated_text_by_index: dict[int, str] = {}
+    by_index: dict[int, str] = {}
     for start in range(0, len(non_empty), _ANTHROPIC_BATCH_SIZE):
-        chunk = non_empty[start : start + _ANTHROPIC_BATCH_SIZE]
-        texts = [seg.text for _, seg in chunk]
-        translated = _translate_batch_via_anthropic(texts, source_language, target_language, api_key)
+        chunk = non_empty[start: start + _ANTHROPIC_BATCH_SIZE]
+        translated = _translate_batch_via_anthropic(
+            [s.text for _, s in chunk], src, tgt, api_key
+        )
         for (i, _), text in zip(chunk, translated):
-            translated_text_by_index[i] = text
+            by_index[i] = text
     return [
         Segment(index=seg.index, start=seg.start, end=seg.end,
-                text=translated_text_by_index.get(i, seg.text))
+                text=by_index.get(i, seg.text))
         for i, seg in enumerate(segments)
     ]
 
@@ -248,14 +240,8 @@ def _translate_segments_via_anthropic(
 
 @lru_cache(maxsize=None)
 def _load_marian_engine(repo_suffix: str):
-    """Load and cache a MarianMT model+tokenizer by repo suffix.
-
-    Keyed by repo suffix (e.g. "en-fa", "en-jap") rather than ISO codes
-    so that aliased pairs (like en-ja -> "en-jap") don't get loaded twice.
-    Returns (tokenizer, model) or raises on failure.
-    """
+    """Load and cache a MarianMT model+tokenizer by repo suffix."""
     from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
-
     model_name = f"Helsinki-NLP/opus-mt-{repo_suffix}"
     try:
         tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -267,37 +253,33 @@ def _load_marian_engine(repo_suffix: str):
     return tokenizer, model
 
 
-def _run_marian_translation(tokenizer, model, text: str) -> str:
-    inputs = tokenizer(text, return_tensors="pt", truncation=True)
+def _run_marian(tokenizer, model, text: str, prefix: Optional[str]) -> str:
+    """Translate one text segment, optionally prepending a language token."""
+    source = f"{prefix} {text}" if prefix else text
+    inputs = tokenizer(source, return_tensors="pt", truncation=True)
     generated = model.generate(**inputs, max_new_tokens=512)
     return tokenizer.decode(generated[0], skip_special_tokens=True).strip()
 
 
 def _translate_segments_via_marian(
-    segments: list[Segment], source_language: str, target_language: str
+    segments: list[Segment], src: str, tgt: str
 ) -> list[Segment]:
-    repo_names = _marian_path_repo_names(source_language, target_language)
-    if not repo_names:
+    steps = _marian_path(src, tgt)
+    if not steps:
         raise TranslationError(
-            f"No Marian translation path known for "
-            f"'{source_language}' -> '{target_language}'."
+            f"No Marian translation path known for '{src}'->'{tgt}'."
         )
-
-    # Load engines (cached after the first call for each repo suffix)
-    engines = [_load_marian_engine(r) for r in repo_names]
-
-    translated_segments = []
+    engines = [(_load_marian_engine(suffix), prefix) for suffix, prefix in steps]
+    result = []
     for seg in segments:
         if not seg.text:
-            translated_segments.append(seg)
+            result.append(seg)
             continue
         text = seg.text
-        for tokenizer, model in engines:
-            text = _run_marian_translation(tokenizer, model, text)
-        translated_segments.append(
-            Segment(index=seg.index, start=seg.start, end=seg.end, text=text)
-        )
-    return translated_segments
+        for (tokenizer, model), prefix in engines:
+            text = _run_marian(tokenizer, model, text, prefix)
+        result.append(Segment(index=seg.index, start=seg.start, end=seg.end, text=text))
+    return result
 
 
 # ---------------------------------------------------------------------------
@@ -305,47 +287,30 @@ def _translate_segments_via_marian(
 # ---------------------------------------------------------------------------
 
 
-class TranslationError(RuntimeError):
-    """Raised when no translation backend/model is available for a pair."""
-
-
 class TranslationService:
-    """Translates a Transcript into one or more target languages.
-
-    The backend is resolved per call from `api_key`, not stored on the
-    instance: pass an Anthropic API key to use Claude for that call, or
-    omit it to use the offline MarianMT backend. This means a single
-    long-lived TranslationService is safe to share across requests/users --
-    nothing about any particular key sticks to it.
-    """
-
     def translate(
         self,
         transcript: Transcript,
         target_language: str,
         api_key: Optional[str] = None,
     ) -> Translation:
-        """Translate every segment of `transcript`, preserving timing."""
         if target_language == transcript.language:
             return Translation(
                 source_language=transcript.language,
                 target_language=target_language,
                 segments=list(transcript.segments),
             )
-
         effective_key = (api_key or "").strip() or os.environ.get("ANTHROPIC_API_KEY")
-
         if effective_key:
-            translated_segments = _translate_segments_via_anthropic(
+            translated = _translate_segments_via_anthropic(
                 transcript.segments, transcript.language, target_language, effective_key
             )
         else:
-            translated_segments = _translate_segments_via_marian(
+            translated = _translate_segments_via_marian(
                 transcript.segments, transcript.language, target_language
             )
-
         return Translation(
             source_language=transcript.language,
             target_language=target_language,
-            segments=translated_segments,
+            segments=translated,
         )