Update app.py

app.py

@@ -1,97 +1,188 @@
 # ============================================================
-# Mesin Translasi Bahasa Jawa → Indonesia → English
-# MODEL: Argos Translate (Offline NMT)
-# CPU SAFE | NO TOKEN | HF SPACES READY
+# FINAL — NLLB + RAG TRANSLATION SYSTEM
+# Jawa → Indonesia → English
 # ============================================================
 
-import subprocess, sys
+import gradio as gr
+import pandas as pd
+import numpy as np
+import faiss
+import torch
 
-def install(pkg):
-    subprocess.check_call(
-        [sys.executable, "-m", "pip", "install", pkg, "--quiet", "--root-user-action=ignore"]
-    )
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+from sentence_transformers import SentenceTransformer
 
-install("pytz")
-install("gradio")
-install("argostranslate")
+# ============================================================
+# Device
+# ============================================================
 
-import gradio as gr
-import argostranslate.package
-import argostranslate.translate
+device = "cuda" if torch.cuda.is_available() else "cpu"
 
 # ============================================================
-# Install language packages (auto-download)
+# Load NLLB Model
 # ============================================================
 
-def install_lang(from_code, to_code):
-    packages = argostranslate.package.get_available_packages()
-    pkg = next(
-        p for p in packages
-        if p.from_code == from_code and p.to_code == to_code
-    )
-    argostranslate.package.install_from_path(pkg.download())
+MODEL_NAME = "facebook/nllb-200-distilled-600M"
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)
+
+model.to(device)
+model.eval()
+
+# ============================================================
+# Language Codes (NLLB official)
+# ============================================================
+
+JAV = "jav_Latn"
+IND = "ind_Latn"
+ENG = "eng_Latn"
+
+# ============================================================
+# Load Knowledge Base (Parallel Corpus)
+# ============================================================
+
+kb = pd.read_csv("kb_jawa_ngoko_krama_indonesia_100k.csv")
+
+kb["jv"] = kb["jv"].astype(str)
+kb["id"] = kb["id"].astype(str)
+
+kb_texts = (
+    "Jawa: " + kb["jv"] +
+    " | Indonesia: " + kb["id"]
+).tolist()
+
+# ============================================================
+# Embedding Model (multilingual)
+# ============================================================
+
+embedder = SentenceTransformer(
+    "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+)
+
+embeddings = embedder.encode(
+    kb_texts,
+    convert_to_numpy=True,
+    show_progress_bar=True
+)
+
+# ============================================================
+# FAISS Index
+# ============================================================
+
+index = faiss.IndexFlatL2(embeddings.shape[1])
+index.add(embeddings)
 
-# Jawa → Indonesia
-install_lang("jv", "id")
+# ============================================================
+# Retrieve Context
+# ============================================================
 
-# Indonesia → English
-install_lang("id", "en")
+def retrieve_context(text, k=3):
+    vec = embedder.encode([text])
+    _, I = index.search(vec, k)
+    return [kb_texts[i] for i in I[0]]
 
 # ============================================================
-# Translation Function
+# Translation Function (RAG-Aware)
 # ============================================================
 
-def translate_jawa(text):
+def translate_javanese(text):
     if not text.strip():
         return "", ""
 
+    # -------------------------------
+    # RAG Retrieval
+    # -------------------------------
+    contexts = retrieve_context(text, k=3)
+
+    rag_prompt = (
+        "Berikut adalah contoh terjemahan bahasa Jawa:\n"
+        + "\n".join(contexts)
+        + "\n\nTerjemahkan kalimat berikut dengan gaya formal dan alami:\n"
+        + text
+    )
+
+    # -------------------------------
     # Jawa → Indonesia
-    indo = argostranslate.translate.translate(
-        text,
-        from_code="jv",
-        to_code="id"
+    # -------------------------------
+    tokenizer.src_lang = JAV
+
+    inputs = tokenizer(
+        rag_prompt,
+        return_tensors="pt",
+        truncation=True,
+        max_length=512
+    ).to(device)
+
+    gen_id = model.generate(
+        **inputs,
+        forced_bos_token_id=tokenizer.convert_tokens_to_ids(IND),
+        max_length=256
     )
 
+    id_text = tokenizer.batch_decode(
+        gen_id,
+        skip_special_tokens=True
+    )[0]
+
+    # -------------------------------
     # Indonesia → English
-    eng = argostranslate.translate.translate(
-        indo,
-        from_code="id",
-        to_code="en"
+    # -------------------------------
+    tokenizer.src_lang = IND
+
+    inputs_en = tokenizer(
+        id_text,
+        return_tensors="pt",
+        truncation=True,
+        max_length=512
+    ).to(device)
+
+    gen_en = model.generate(
+        **inputs_en,
+        forced_bos_token_id=tokenizer.convert_tokens_to_ids(ENG),
+        max_length=256
     )
 
-    return indo, eng
+    en_text = tokenizer.batch_decode(
+        gen_en,
+        skip_special_tokens=True
+    )[0]
+
+    return id_text, en_text
 
 # ============================================================
 # Gradio UI
 # ============================================================
 
-with gr.Blocks(title="Mesin Translasi Bahasa Jawa") as demo:
+with gr.Blocks(title="NLLB + RAG Javanese Translator") as demo:
+
     gr.Markdown("""
-    ## 🌾 Mesin Translasi Bahasa Jawa
-    **Jawa → Indonesia → English**
-
-    ✔ Offline Neural MT (Argos)  
-    ✔ Tanpa API Key  
-    ✔ CPU-only  
-    ✔ Stabil & deterministik  
-    ✔ Cocok untuk penelitian & produksi
+    ## 🌾 Mesin Translasi Bahasa Jawa  
+    **NLLB + Retrieval-Augmented Generation**
+
+    ✔ Parallel corpus 100.000 kalimat  
+    ✔ Ngoko + Krama  
+    ✔ Tanpa API key  
+    ✔ Cocok penelitian, tesis, jurnal  
     """)
 
     input_text = gr.Textbox(
         label="Input Bahasa Jawa",
-        placeholder="Contoh: Aku seneng sinau basa Jawa amarga iku warisan budaya.",
+        placeholder="Kula badhe sowan dhateng griya eyang sesuk.",
         lines=4
     )
 
     with gr.Row():
-        output_id = gr.Textbox(label="Bahasa Indonesia", lines=4)
-        output_en = gr.Textbox(label="English", lines=4)
+        output_id = gr.Textbox(label="Terjemahan Bahasa Indonesia", lines=4)
+        output_en = gr.Textbox(label="Translation to English", lines=4)
 
     btn = gr.Button("🔄 Terjemahkan")
+
     btn.click(
-        fn=translate_jawa,
+        fn=translate_javanese,
         inputs=input_text,
         outputs=[output_id, output_en]
     )
 
 demo.launch()
+