Spaces:
Sleeping
Sleeping
usertea commited on
Commit ·
815d3aa
1
Parent(s): 1788966
Helsinki-NLP published the English→Japanese model as opus-mt-en-jap - using jap, not the standard code ja - while the reverse direction (opus-mt-ja-en) correctly uses ja. That's an inconsistency in their own catalog, not my logic, but my code assumed standard ISO codes everywhere, so it was 404ing on a model that actually exists.
Browse files- app.py +1 -1
- services/translation.py +24 -6
app.py
CHANGED
|
@@ -325,7 +325,7 @@ with gr.Blocks(title="EchoScript") as demo:
|
|
| 325 |
**Upload Audio → Select Audio Window → Detect Language & Generate Transcript
|
| 326 |
→ Preview & Choose Languages → Generate Translations → Copy / Download**
|
| 327 |
|
| 328 |
-
<sub>build: 2026-06-
|
| 329 |
"""
|
| 330 |
)
|
| 331 |
|
|
|
|
| 325 |
**Upload Audio → Select Audio Window → Detect Language & Generate Transcript
|
| 326 |
→ Preview & Choose Languages → Generate Translations → Copy / Download**
|
| 327 |
|
| 328 |
+
<sub>build: 2026-06-28 22:17 UTC · fixed en-jap Helsinki-NLP naming exception</sub>
|
| 329 |
"""
|
| 330 |
)
|
| 331 |
|
services/translation.py
CHANGED
|
@@ -24,12 +24,13 @@ whether an Anthropic API key was supplied -- never stored:
|
|
| 24 |
through to the Anthropic client for that one call and is never written
|
| 25 |
to disk, logged, or cached in any module-level state.
|
| 26 |
- "marian": fully offline, no API key needed. Uses local Helsinki-NLP
|
| 27 |
-
MarianMT models via `transformers`
|
| 28 |
-
|
|
|
|
| 29 |
|
| 30 |
The UI is expected to only offer the larger ANTHROPIC_TARGET_LANGUAGES
|
| 31 |
-
list once a key has been entered, and
|
| 32 |
-
see app.py.
|
| 33 |
"""
|
| 34 |
|
| 35 |
from __future__ import annotations
|
|
@@ -195,7 +196,7 @@ def _try_load_marian_engine(source_language: str, target_language: str):
|
|
| 195 |
"""
|
| 196 |
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer # heavy import, deferred
|
| 197 |
|
| 198 |
-
model_name =
|
| 199 |
try:
|
| 200 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
| 201 |
model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
|
|
@@ -204,6 +205,23 @@ def _try_load_marian_engine(source_language: str, target_language: str):
|
|
| 204 |
return tokenizer, model
|
| 205 |
|
| 206 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
@lru_cache(maxsize=None)
|
| 208 |
def _marian_model_exists(source_language: str, target_language: str) -> bool:
|
| 209 |
"""Check (and cache) whether Helsinki-NLP publishes this exact pair.
|
|
@@ -222,7 +240,7 @@ def _marian_model_exists(source_language: str, target_language: str) -> bool:
|
|
| 222 |
from huggingface_hub.utils import HfHubHTTPError
|
| 223 |
|
| 224 |
try:
|
| 225 |
-
HfApi().model_info(
|
| 226 |
return True
|
| 227 |
except HfHubHTTPError:
|
| 228 |
return False
|
|
|
|
| 24 |
through to the Anthropic client for that one call and is never written
|
| 25 |
to disk, logged, or cached in any module-level state.
|
| 26 |
- "marian": fully offline, no API key needed. Uses local Helsinki-NLP
|
| 27 |
+
MarianMT models via `transformers`. Coverage is checked for real, per
|
| 28 |
+
source language, against the Hub (direct pair or English pivot) -- see
|
| 29 |
+
available_marian_targets() -- rather than assumed from a fixed list.
|
| 30 |
|
| 31 |
The UI is expected to only offer the larger ANTHROPIC_TARGET_LANGUAGES
|
| 32 |
+
list once a key has been entered, and available_marian_targets(source)
|
| 33 |
+
otherwise -- see app.py.
|
| 34 |
"""
|
| 35 |
|
| 36 |
from __future__ import annotations
|
|
|
|
| 196 |
"""
|
| 197 |
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer # heavy import, deferred
|
| 198 |
|
| 199 |
+
model_name = _marian_repo_id(source_language, target_language)
|
| 200 |
try:
|
| 201 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
| 202 |
model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
|
|
|
|
| 205 |
return tokenizer, model
|
| 206 |
|
| 207 |
|
| 208 |
+
# Helsinki-NLP doesn't always use standard ISO 639-1 codes in its repo
|
| 209 |
+
# names. Known exceptions go here, mapped to the actual code used in that
|
| 210 |
+
# specific repo name -- e.g. the en->Japanese model is published as
|
| 211 |
+
# "Helsinki-NLP/opus-mt-en-jap" (not "...-en-ja"), even though the reverse
|
| 212 |
+
# direction "Helsinki-NLP/opus-mt-ja-en" correctly uses "ja". Without this,
|
| 213 |
+
# every availability check for Japanese-as-a-target would 404 on a model
|
| 214 |
+
# that actually exists, regardless of source language.
|
| 215 |
+
_MARIAN_CODE_ALIASES: dict[tuple[str, str], str] = {
|
| 216 |
+
("en", "ja"): "jap",
|
| 217 |
+
}
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _marian_repo_id(source_language: str, target_language: str) -> str:
|
| 221 |
+
aliased_target = _MARIAN_CODE_ALIASES.get((source_language, target_language), target_language)
|
| 222 |
+
return f"Helsinki-NLP/opus-mt-{source_language}-{aliased_target}"
|
| 223 |
+
|
| 224 |
+
|
| 225 |
@lru_cache(maxsize=None)
|
| 226 |
def _marian_model_exists(source_language: str, target_language: str) -> bool:
|
| 227 |
"""Check (and cache) whether Helsinki-NLP publishes this exact pair.
|
|
|
|
| 240 |
from huggingface_hub.utils import HfHubHTTPError
|
| 241 |
|
| 242 |
try:
|
| 243 |
+
HfApi().model_info(_marian_repo_id(source_language, target_language))
|
| 244 |
return True
|
| 245 |
except HfHubHTTPError:
|
| 246 |
return False
|