usertea commited on
Commit
815d3aa
·
1 Parent(s): 1788966

Helsinki-NLP published the English→Japanese model as opus-mt-en-jap - using jap, not the standard code ja - while the reverse direction (opus-mt-ja-en) correctly uses ja. That's an inconsistency in their own catalog, not my logic, but my code assumed standard ISO codes everywhere, so it was 404ing on a model that actually exists.

Browse files
Files changed (2) hide show
  1. app.py +1 -1
  2. services/translation.py +24 -6
app.py CHANGED
@@ -325,7 +325,7 @@ with gr.Blocks(title="EchoScript") as demo:
325
  **Upload Audio → Select Audio Window → Detect Language & Generate Transcript
326
  → Preview & Choose Languages → Generate Translations → Copy / Download**
327
 
328
- <sub>build: 2026-06-27 00:24 UTC &middot; real per-source-language Marian availability (no more fixed 4-language guess)</sub>
329
  """
330
  )
331
 
 
325
  **Upload Audio &rarr; Select Audio Window &rarr; Detect Language & Generate Transcript
326
  &rarr; Preview & Choose Languages &rarr; Generate Translations &rarr; Copy / Download**
327
 
328
+ <sub>build: 2026-06-28 22:17 UTC &middot; fixed en-jap Helsinki-NLP naming exception</sub>
329
  """
330
  )
331
 
services/translation.py CHANGED
@@ -24,12 +24,13 @@ whether an Anthropic API key was supplied -- never stored:
24
  through to the Anthropic client for that one call and is never written
25
  to disk, logged, or cached in any module-level state.
26
  - "marian": fully offline, no API key needed. Uses local Helsinki-NLP
27
- MarianMT models via `transformers`, restricted to the small set of
28
- languages where direct or English-pivoted coverage is reliable.
 
29
 
30
  The UI is expected to only offer the larger ANTHROPIC_TARGET_LANGUAGES
31
- list once a key has been entered, and MARIAN_TARGET_LANGUAGES otherwise --
32
- see app.py.
33
  """
34
 
35
  from __future__ import annotations
@@ -195,7 +196,7 @@ def _try_load_marian_engine(source_language: str, target_language: str):
195
  """
196
  from transformers import AutoModelForSeq2SeqLM, AutoTokenizer # heavy import, deferred
197
 
198
- model_name = f"Helsinki-NLP/opus-mt-{source_language}-{target_language}"
199
  try:
200
  tokenizer = AutoTokenizer.from_pretrained(model_name)
201
  model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
@@ -204,6 +205,23 @@ def _try_load_marian_engine(source_language: str, target_language: str):
204
  return tokenizer, model
205
 
206
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
207
  @lru_cache(maxsize=None)
208
  def _marian_model_exists(source_language: str, target_language: str) -> bool:
209
  """Check (and cache) whether Helsinki-NLP publishes this exact pair.
@@ -222,7 +240,7 @@ def _marian_model_exists(source_language: str, target_language: str) -> bool:
222
  from huggingface_hub.utils import HfHubHTTPError
223
 
224
  try:
225
- HfApi().model_info(f"Helsinki-NLP/opus-mt-{source_language}-{target_language}")
226
  return True
227
  except HfHubHTTPError:
228
  return False
 
24
  through to the Anthropic client for that one call and is never written
25
  to disk, logged, or cached in any module-level state.
26
  - "marian": fully offline, no API key needed. Uses local Helsinki-NLP
27
+ MarianMT models via `transformers`. Coverage is checked for real, per
28
+ source language, against the Hub (direct pair or English pivot) -- see
29
+ available_marian_targets() -- rather than assumed from a fixed list.
30
 
31
  The UI is expected to only offer the larger ANTHROPIC_TARGET_LANGUAGES
32
+ list once a key has been entered, and available_marian_targets(source)
33
+ otherwise -- see app.py.
34
  """
35
 
36
  from __future__ import annotations
 
196
  """
197
  from transformers import AutoModelForSeq2SeqLM, AutoTokenizer # heavy import, deferred
198
 
199
+ model_name = _marian_repo_id(source_language, target_language)
200
  try:
201
  tokenizer = AutoTokenizer.from_pretrained(model_name)
202
  model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
 
205
  return tokenizer, model
206
 
207
 
208
+ # Helsinki-NLP doesn't always use standard ISO 639-1 codes in its repo
209
+ # names. Known exceptions go here, mapped to the actual code used in that
210
+ # specific repo name -- e.g. the en->Japanese model is published as
211
+ # "Helsinki-NLP/opus-mt-en-jap" (not "...-en-ja"), even though the reverse
212
+ # direction "Helsinki-NLP/opus-mt-ja-en" correctly uses "ja". Without this,
213
+ # every availability check for Japanese-as-a-target would 404 on a model
214
+ # that actually exists, regardless of source language.
215
+ _MARIAN_CODE_ALIASES: dict[tuple[str, str], str] = {
216
+ ("en", "ja"): "jap",
217
+ }
218
+
219
+
220
+ def _marian_repo_id(source_language: str, target_language: str) -> str:
221
+ aliased_target = _MARIAN_CODE_ALIASES.get((source_language, target_language), target_language)
222
+ return f"Helsinki-NLP/opus-mt-{source_language}-{aliased_target}"
223
+
224
+
225
  @lru_cache(maxsize=None)
226
  def _marian_model_exists(source_language: str, target_language: str) -> bool:
227
  """Check (and cache) whether Helsinki-NLP publishes this exact pair.
 
240
  from huggingface_hub.utils import HfHubHTTPError
241
 
242
  try:
243
+ HfApi().model_info(_marian_repo_id(source_language, target_language))
244
  return True
245
  except HfHubHTTPError:
246
  return False