arindae commited on
Commit
7ffed06
·
verified ·
1 Parent(s): cbf2f35

Added sunbird provider

Browse files
backend/providers/__init__.py CHANGED
@@ -13,7 +13,7 @@ from providers.base import ProviderInfo, TranslationProvider
13
  from providers.gemini import GeminiProvider
14
  from providers.groq import GroqProvider, GroqQwenProvider
15
  from providers.madlad import MADLADProvider
16
- from providers.nllb import NLLBProvider
17
  from providers.ollama import OllamaProvider
18
 
19
  _PROVIDERS: list[TranslationProvider] = [
@@ -26,6 +26,7 @@ _PROVIDERS: list[TranslationProvider] = [
26
  ),
27
  hf_model=os.environ.get("HF_MODEL_1_3B", "facebook/nllb-200-distilled-1.3B"),
28
  ),
 
29
  MADLADProvider(),
30
  OllamaProvider(),
31
  GeminiProvider(),
 
13
  from providers.gemini import GeminiProvider
14
  from providers.groq import GroqProvider, GroqQwenProvider
15
  from providers.madlad import MADLADProvider
16
+ from providers.nllb import NLLBProvider, SunbirdNLLBProvider
17
  from providers.ollama import OllamaProvider
18
 
19
  _PROVIDERS: list[TranslationProvider] = [
 
26
  ),
27
  hf_model=os.environ.get("HF_MODEL_1_3B", "facebook/nllb-200-distilled-1.3B"),
28
  ),
29
+ SunbirdNLLBProvider(),
30
  MADLADProvider(),
31
  OllamaProvider(),
32
  GeminiProvider(),
backend/providers/nllb.py CHANGED
@@ -1,8 +1,4 @@
1
- """NLLB-200 engine — local inference via CTranslate2 (int8).
2
-
3
- Light enough to run on a free HuggingFace Spaces CPU. Heavy objects load lazily
4
- on first use and are shared across requests.
5
- """
6
 
7
  from __future__ import annotations
8
 
@@ -81,3 +77,67 @@ class NLLBProvider(TranslationProvider):
81
  for line in text.split("\n"):
82
  out.append(self._translate_line(line, src, tgt) if line.strip() else "")
83
  return "\n".join(out)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """NLLB-200 & Sunbird NLLB engines — local inference via CTranslate2 (int8)."""
 
 
 
 
2
 
3
  from __future__ import annotations
4
 
 
77
  for line in text.split("\n"):
78
  out.append(self._translate_line(line, src, tgt) if line.strip() else "")
79
  return "\n".join(out)
80
+
81
+
82
+ # Sunbird-specific language code mapping
83
+ SUNBIRD_LANGUAGES = {
84
+ "eng": "English",
85
+ "ach": "Acholi",
86
+ "lgg": "Lugbara",
87
+ "lug": "Luganda",
88
+ "nyn": "Runyankole",
89
+ "teo": "Ateso",
90
+ }
91
+
92
+ # Auto-map standard FLORES-200 codes to Sunbird 3-letter codes
93
+ SUNBIRD_CODE_MAP = {
94
+ "eng_Latn": "eng",
95
+ "ach_Latn": "ach",
96
+ "lgg_Latn": "lgg",
97
+ "lug_Latn": "lug",
98
+ "nyn_Latn": "nyn",
99
+ "teo_Latn": "teo",
100
+ }
101
+
102
+
103
+ class SunbirdNLLBProvider(NLLBProvider):
104
+ description = "Sunbird NLLB-200 (3.3B) fine-tune · Ugandan languages & English"
105
+
106
+ def __init__(
107
+ self,
108
+ provider_id: str = "sunbird_3_3b",
109
+ name: str = "Sunbird NLLB (3.3B)",
110
+ model_dir: str = os.environ.get(
111
+ "CT2_MODEL_DIR_SUNBIRD", "/data/models/sunbird-nllb-3.3b-int8"
112
+ ),
113
+ hf_model: str = os.environ.get(
114
+ "HF_MODEL_SUNBIRD", "Sunbird/translate-nllb-3.3b-salt"
115
+ ),
116
+ ) -> None:
117
+ super().__init__(
118
+ provider_id=provider_id,
119
+ name=name,
120
+ model_dir=model_dir,
121
+ hf_model=hf_model,
122
+ )
123
+
124
+ def _normalize_lang(self, code: str) -> str:
125
+ """Converts FLORES codes (e.g. 'lug_Latn') or 3-letter codes ('lug') to Sunbird format."""
126
+ if code in SUNBIRD_CODE_MAP:
127
+ return SUNBIRD_CODE_MAP[code]
128
+ if code in SUNBIRD_LANGUAGES:
129
+ return code
130
+ raise ValueError(
131
+ f"Unsupported language code '{code}' for Sunbird NLLB. "
132
+ f"Supported codes: {list(SUNBIRD_LANGUAGES.keys())}"
133
+ )
134
+
135
+ def translate(self, text: str, src: str, tgt: str, api_key: str | None = None) -> str:
136
+ src_code = self._normalize_lang(src)
137
+ tgt_code = self._normalize_lang(tgt)
138
+
139
+ self._ensure_loaded()
140
+ out = []
141
+ for line in text.split("\n"):
142
+ out.append(self._translate_line(line, src_code, tgt_code) if line.strip() else "")
143
+ return "\n".join(out)