Spaces:
Sleeping
Sleeping
Added sunbird provider
Browse files- backend/providers/__init__.py +2 -1
- backend/providers/nllb.py +65 -5
backend/providers/__init__.py
CHANGED
|
@@ -13,7 +13,7 @@ from providers.base import ProviderInfo, TranslationProvider
|
|
| 13 |
from providers.gemini import GeminiProvider
|
| 14 |
from providers.groq import GroqProvider, GroqQwenProvider
|
| 15 |
from providers.madlad import MADLADProvider
|
| 16 |
-
from providers.nllb import NLLBProvider
|
| 17 |
from providers.ollama import OllamaProvider
|
| 18 |
|
| 19 |
_PROVIDERS: list[TranslationProvider] = [
|
|
@@ -26,6 +26,7 @@ _PROVIDERS: list[TranslationProvider] = [
|
|
| 26 |
),
|
| 27 |
hf_model=os.environ.get("HF_MODEL_1_3B", "facebook/nllb-200-distilled-1.3B"),
|
| 28 |
),
|
|
|
|
| 29 |
MADLADProvider(),
|
| 30 |
OllamaProvider(),
|
| 31 |
GeminiProvider(),
|
|
|
|
| 13 |
from providers.gemini import GeminiProvider
|
| 14 |
from providers.groq import GroqProvider, GroqQwenProvider
|
| 15 |
from providers.madlad import MADLADProvider
|
| 16 |
+
from providers.nllb import NLLBProvider, SunbirdNLLBProvider
|
| 17 |
from providers.ollama import OllamaProvider
|
| 18 |
|
| 19 |
_PROVIDERS: list[TranslationProvider] = [
|
|
|
|
| 26 |
),
|
| 27 |
hf_model=os.environ.get("HF_MODEL_1_3B", "facebook/nllb-200-distilled-1.3B"),
|
| 28 |
),
|
| 29 |
+
SunbirdNLLBProvider(),
|
| 30 |
MADLADProvider(),
|
| 31 |
OllamaProvider(),
|
| 32 |
GeminiProvider(),
|
backend/providers/nllb.py
CHANGED
|
@@ -1,8 +1,4 @@
|
|
| 1 |
-
"""NLLB-200
|
| 2 |
-
|
| 3 |
-
Light enough to run on a free HuggingFace Spaces CPU. Heavy objects load lazily
|
| 4 |
-
on first use and are shared across requests.
|
| 5 |
-
"""
|
| 6 |
|
| 7 |
from __future__ import annotations
|
| 8 |
|
|
@@ -81,3 +77,67 @@ class NLLBProvider(TranslationProvider):
|
|
| 81 |
for line in text.split("\n"):
|
| 82 |
out.append(self._translate_line(line, src, tgt) if line.strip() else "")
|
| 83 |
return "\n".join(out)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""NLLB-200 & Sunbird NLLB engines — local inference via CTranslate2 (int8)."""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
from __future__ import annotations
|
| 4 |
|
|
|
|
| 77 |
for line in text.split("\n"):
|
| 78 |
out.append(self._translate_line(line, src, tgt) if line.strip() else "")
|
| 79 |
return "\n".join(out)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
# Sunbird-specific language code mapping
|
| 83 |
+
SUNBIRD_LANGUAGES = {
|
| 84 |
+
"eng": "English",
|
| 85 |
+
"ach": "Acholi",
|
| 86 |
+
"lgg": "Lugbara",
|
| 87 |
+
"lug": "Luganda",
|
| 88 |
+
"nyn": "Runyankole",
|
| 89 |
+
"teo": "Ateso",
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
# Auto-map standard FLORES-200 codes to Sunbird 3-letter codes
|
| 93 |
+
SUNBIRD_CODE_MAP = {
|
| 94 |
+
"eng_Latn": "eng",
|
| 95 |
+
"ach_Latn": "ach",
|
| 96 |
+
"lgg_Latn": "lgg",
|
| 97 |
+
"lug_Latn": "lug",
|
| 98 |
+
"nyn_Latn": "nyn",
|
| 99 |
+
"teo_Latn": "teo",
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
class SunbirdNLLBProvider(NLLBProvider):
|
| 104 |
+
description = "Sunbird NLLB-200 (3.3B) fine-tune · Ugandan languages & English"
|
| 105 |
+
|
| 106 |
+
def __init__(
|
| 107 |
+
self,
|
| 108 |
+
provider_id: str = "sunbird_3_3b",
|
| 109 |
+
name: str = "Sunbird NLLB (3.3B)",
|
| 110 |
+
model_dir: str = os.environ.get(
|
| 111 |
+
"CT2_MODEL_DIR_SUNBIRD", "/data/models/sunbird-nllb-3.3b-int8"
|
| 112 |
+
),
|
| 113 |
+
hf_model: str = os.environ.get(
|
| 114 |
+
"HF_MODEL_SUNBIRD", "Sunbird/translate-nllb-3.3b-salt"
|
| 115 |
+
),
|
| 116 |
+
) -> None:
|
| 117 |
+
super().__init__(
|
| 118 |
+
provider_id=provider_id,
|
| 119 |
+
name=name,
|
| 120 |
+
model_dir=model_dir,
|
| 121 |
+
hf_model=hf_model,
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
def _normalize_lang(self, code: str) -> str:
|
| 125 |
+
"""Converts FLORES codes (e.g. 'lug_Latn') or 3-letter codes ('lug') to Sunbird format."""
|
| 126 |
+
if code in SUNBIRD_CODE_MAP:
|
| 127 |
+
return SUNBIRD_CODE_MAP[code]
|
| 128 |
+
if code in SUNBIRD_LANGUAGES:
|
| 129 |
+
return code
|
| 130 |
+
raise ValueError(
|
| 131 |
+
f"Unsupported language code '{code}' for Sunbird NLLB. "
|
| 132 |
+
f"Supported codes: {list(SUNBIRD_LANGUAGES.keys())}"
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
def translate(self, text: str, src: str, tgt: str, api_key: str | None = None) -> str:
|
| 136 |
+
src_code = self._normalize_lang(src)
|
| 137 |
+
tgt_code = self._normalize_lang(tgt)
|
| 138 |
+
|
| 139 |
+
self._ensure_loaded()
|
| 140 |
+
out = []
|
| 141 |
+
for line in text.split("\n"):
|
| 142 |
+
out.append(self._translate_line(line, src_code, tgt_code) if line.strip() else "")
|
| 143 |
+
return "\n".join(out)
|