Add Formosan-Chinese directional models to MT demo

README.md

@@ -1,5 +1,5 @@
 ---
-title: Formosan ↔ English MT
+title: Formosan ↔ English / Chinese MT
 emoji: 🌿
 colorFrom: yellow
 colorTo: green
@@ -8,10 +8,12 @@ sdk_version: 5.49.1
 app_file: app.py
 pinned: false
 license: cc-by-nc-4.0
-short_description: Formosan-English NLLB-200 translation demo
+short_description: Formosan-English/Chinese NLLB-200 translation demo
 models:
 - FormosanBank/nllb200-formosan-en-spm8k
 - FormosanBank/nllb200-en-formosan-spm8k
+- FormosanBank/nllb200-formosan-zh-spm8k
+- FormosanBank/nllb200-zh-formosan-spm8k
 tags:
 - translation
 - nllb
@@ -20,12 +22,14 @@ tags:
 - endangered-languages
 ---
 
-# Formosan ↔ English Machine Translation
+# Formosan ↔ English / Chinese Machine Translation
 
 This Space is a research demo for FormosanBank directional NLLB-200 models:
 
 - [`FormosanBank/nllb200-formosan-en-spm8k`](https://huggingface.co/FormosanBank/nllb200-formosan-en-spm8k)
 - [`FormosanBank/nllb200-en-formosan-spm8k`](https://huggingface.co/FormosanBank/nllb200-en-formosan-spm8k)
+- [`FormosanBank/nllb200-formosan-zh-spm8k`](https://huggingface.co/FormosanBank/nllb200-formosan-zh-spm8k)
+- [`FormosanBank/nllb200-zh-formosan-spm8k`](https://huggingface.co/FormosanBank/nllb200-zh-formosan-spm8k)
 
 The interface hides the metadata control tags used during training. Users only choose direction, language, and optionally source/domain and dialect metadata.
 

app.py

@@ -27,7 +27,10 @@ except Exception:
 
 F2EN_MODEL_ID = "FormosanBank/nllb200-formosan-en-spm8k"
 EN2F_MODEL_ID = "FormosanBank/nllb200-en-formosan-spm8k"
+F2ZH_MODEL_ID = "FormosanBank/nllb200-formosan-zh-spm8k"
+ZH2F_MODEL_ID = "FormosanBank/nllb200-zh-formosan-spm8k"
 ENGLISH_LID = "eng_Latn"
+CHINESE_LID = "zho_Hant"
 MAX_INPUT_LENGTH = 384
 
 
@@ -52,6 +55,8 @@ FORMOSAN_LANGS: Dict[str, Tuple[str, str]] = {
 DIRECTION_LABELS = {
     "Formosan → English": "f2en",
     "English → Formosan": "en2f",
+    "Formosan → Chinese": "f2zh",
+    "Chinese → Formosan": "zh2f",
 }
 
 DOMAIN_CHOICES = {
@@ -155,6 +160,26 @@ EXAMPLE_PRESETS = {
         4,
         1.15,
     ),
+    "Chinese → Amis: 他回家了。": (
+        "他回家了。",
+        "Chinese → Formosan",
+        "Amis",
+        "Unknown / general",
+        "Default / unknown",
+        96,
+        4,
+        1.15,
+    ),
+    "Amis → Chinese: Pa'araw cingra...": (
+        "Pa'araw cingra to demak nira.",
+        "Formosan → Chinese",
+        "Amis",
+        "Unknown / general",
+        "Default / unknown",
+        96,
+        4,
+        1.15,
+    ),
 }
 
 
@@ -190,6 +215,10 @@ def preproc_formosan(text: str) -> str:
     return unicodedata.normalize("NFKC", replace_nonprint(text)).strip()
 
 
+def preproc_chinese(text: str) -> str:
+    return unicodedata.normalize("NFKC", replace_nonprint(text)).strip()
+
+
 @dataclass
 class ModelBundle:
     tokenizer: NllbTokenizer
@@ -206,7 +235,12 @@ def active_device() -> torch.device:
 
 
 def model_id_for(direction_key: str) -> str:
-    return F2EN_MODEL_ID if direction_key == "f2en" else EN2F_MODEL_ID
+    return {
+        "f2en": F2EN_MODEL_ID,
+        "en2f": EN2F_MODEL_ID,
+        "f2zh": F2ZH_MODEL_ID,
+        "zh2f": ZH2F_MODEL_ID,
+    }[direction_key]
 
 
 def load_bundle(direction_key: str) -> ModelBundle:
@@ -214,6 +248,11 @@ def load_bundle(direction_key: str) -> ModelBundle:
     device = active_device()
     with MODEL_LOCK:
         if direction_key not in MODEL_CACHE:
+            if device.type == "cuda":
+                for bundle in MODEL_CACHE.values():
+                    if next(bundle.model.parameters()).device.type == "cuda":
+                        bundle.model.to("cpu")
+                torch.cuda.empty_cache()
             tokenizer = NllbTokenizer.from_pretrained(repo_id)
             dtype = torch.float16 if device.type == "cuda" else torch.float32
             model = AutoModelForSeq2SeqLM.from_pretrained(repo_id, torch_dtype=dtype)
@@ -255,7 +294,11 @@ def format_prompt(
     dialect_tag = known_tag(tokenizer, f"<dialect_{dialect_value}>", "<dialect_default>")
     if direction_key == "f2en":
         return f"<to_eng> <src_{lang_code}> {domain_tag} {dialect_tag} {text}"
-    return f"<to_{lang_code}> <src_eng> {domain_tag} {dialect_tag} {text}"
+    if direction_key == "en2f":
+        return f"<to_{lang_code}> <src_eng> {domain_tag} {dialect_tag} {text}"
+    if direction_key == "f2zh":
+        return f"<to_zh> <src_{lang_code}> {domain_tag} {dialect_tag} {text}"
+    return f"<to_{lang_code}> <src_zh> {domain_tag} {dialect_tag} {text}"
 
 
 @gpu
@@ -285,10 +328,18 @@ def translate(
         tokenizer.src_lang = lang_lid
         clean_text = preproc_formosan(raw_text)
         target_lid = ENGLISH_LID
-    else:
+    elif direction_key == "en2f":
         tokenizer.src_lang = ENGLISH_LID
         clean_text = preproc_english(raw_text)
         target_lid = lang_lid
+    elif direction_key == "f2zh":
+        tokenizer.src_lang = lang_lid
+        clean_text = preproc_formosan(raw_text)
+        target_lid = CHINESE_LID
+    else:
+        tokenizer.src_lang = CHINESE_LID
+        clean_text = preproc_chinese(raw_text)
+        target_lid = lang_lid
 
     prompt = format_prompt(tokenizer, clean_text, direction_key, lang_code, domain_value, dialect_value)
     forced_bos = tokenizer.convert_tokens_to_ids(target_lid)
@@ -327,14 +378,17 @@ def translate(
 
 
 def swap_placeholder(direction_label: str, formosan_language: str) -> gr.Textbox:
-    if DIRECTION_LABELS[direction_label] == "f2en":
+    direction_key = DIRECTION_LABELS[direction_label]
+    if direction_key in {"f2en", "f2zh"}:
+        target = "English" if direction_key == "f2en" else "Traditional Chinese"
         return gr.Textbox(
-            placeholder=f"Enter text in {formosan_language}. The app will add the hidden NLLB/control tags.",
+            placeholder=f"Enter text in {formosan_language}. The app will translate it into {target}.",
             label=f"{formosan_language} input",
         )
+    source = "English" if direction_key == "en2f" else "Traditional Chinese"
     return gr.Textbox(
-        placeholder=f"Enter English text to translate into {formosan_language}.",
-        label="English input",
+        placeholder=f"Enter {source} text to translate into {formosan_language}.",
+        label=f"{source} input",
     )
 
 
@@ -346,9 +400,9 @@ def load_example(example_name: str):
 with gr.Blocks(title="FormosanBank MT") as demo:
     gr.Markdown(
         """
-# Formosan ↔ English MT
+# Formosan ↔ English / Chinese MT
 
-Translate between English and 15 Formosan languages using directional NLLB-200 checkpoints.
+Translate between 15 Formosan languages and English or Traditional Chinese using directional NLLB-200 checkpoints.
 The app adds the training control tags internally; users only choose direction and language.
         """
     )
@@ -428,7 +482,9 @@ The app adds the training control tags internally; users only choose direction a
 **Current hard-split scores**
 
 Formosan→English: BLEU 8.23 / chrF2 27.35  
-English→Formosan: BLEU 5.77 / chrF2 30.24
+English→Formosan: BLEU 5.77 / chrF2 30.24  
+Formosan→Chinese: BLEU 9.79 / chrF2 11.77  
+Chinese→Formosan: BLEU 7.65 / chrF2 32.97
                 """
             )
 
@@ -440,9 +496,12 @@ This is a research demo, not an authoritative translation service. Outputs can b
 or culturally inappropriate, especially when translating from English into a Formosan language.
 Use fluent-speaker review for community-facing, ceremonial, legal, medical, or other high-stakes use.
 
-The model cards and evaluation details are available at
-[`FormosanBank/nllb200-formosan-en-spm8k`](https://huggingface.co/FormosanBank/nllb200-formosan-en-spm8k)
-and [`FormosanBank/nllb200-en-formosan-spm8k`](https://huggingface.co/FormosanBank/nllb200-en-formosan-spm8k).
+Model cards and evaluation details are available at:
+
+- [`FormosanBank/nllb200-formosan-en-spm8k`](https://huggingface.co/FormosanBank/nllb200-formosan-en-spm8k)
+- [`FormosanBank/nllb200-en-formosan-spm8k`](https://huggingface.co/FormosanBank/nllb200-en-formosan-spm8k)
+- [`FormosanBank/nllb200-formosan-zh-spm8k`](https://huggingface.co/FormosanBank/nllb200-formosan-zh-spm8k)
+- [`FormosanBank/nllb200-zh-formosan-spm8k`](https://huggingface.co/FormosanBank/nllb200-zh-formosan-spm8k)
         """
     )