Upload 10 files

.gitattributes

@@ -1,35 +1,36 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+docs/Evaluate[[:space:]]Machine[[:space:]]Translation[[:space:]]using[[:space:]]Automatic[[:space:]]Metrics.pdf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,45 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+.Python
+*.egg-info/
+dist/
+build/
+*.egg
+
+# Virtual environments
+venv/
+.venv/
+env/
+
+# Jupyter
+.ipynb_checkpoints/
+*.ipynb_checkpoints
+
+# Model cache (HuggingFace downloads)
+~/.cache/huggingface/
+*.bin
+*.safetensors
+*.h5
+
+# Data files
+*.npy
+*.pkl
+*.csv
+*.tsv
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Gradio temp files
+gradio_cached_examples/
+flagged/

README.md

@@ -1,13 +1,361 @@
----
-title: Machine Translation
-emoji: 🐠
-colorFrom: yellow
-colorTo: yellow
-sdk: gradio
-sdk_version: 6.18.0
-python_version: '3.13'
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: English → Indian Languages Machine Translation
+emoji: 🌐
+colorFrom: green
+colorTo: purple
+sdk: gradio
+sdk_version: 6.12.0
+app_file: app.py
+pinned: false
+---
+
+# 🌐 English → Indian Languages Machine Translation
+
+A complete machine translation pipeline for **English → Indian languages** using the **NLLB-200 multilingual model**, evaluated with multiple automatic metrics and deployed via an interactive **Gradio web application**.
+
+---
+
+# 📋 Table of Contents
+
+- Overview
+- Languages Supported
+- Dataset
+- Models Evaluated
+- Evaluation Results
+- Cross-Language Evaluation
+- Metric Descriptions
+- Multilingual Extension Task
+- Project Structure
+- Quick Start
+- Running the App
+- Running Evaluation
+- Deployment Architecture
+- Report
+
+---
+
+# Overview
+
+This project evaluates multiple pretrained multilingual translation models on **English → Tamil** (primary evaluation) and demonstrates multilingual capability across multiple Indian languages using the best performing model.
+
+The best performing model was:
+
+```
+facebook/nllb-200-distilled-600M
+```
+
+This model supports **200+ languages**, allowing the application to translate English sentences into multiple Indian languages using a single model.
+
+Key observation:
+
+- NLLB-200 significantly outperforms **M2M100** and **T5-Base** for English → Tamil translation.
+- The same model can also perform **zero-shot translation** for other Indian languages.
+
+---
+
+# Languages Supported
+
+| Language | Script | NLLB Token | BERTScore Lang |
+|--------|--------|------------|---------------|
+| Tamil | தமிழ் | `tam_Taml` | ta |
+| Hindi | हिन्दी | `hin_Deva` | hi |
+| Telugu | తెలుగు | `tel_Telu` | te |
+| Kannada | ಕನ್ನಡ | `kan_Knda` | kn |
+| Malayalam | മലയാളം | `mal_Mlym` | ml |
+
+Switching languages in the application **does not reload the model**.  
+Only the **target language token (`forced_bos_token_id`)** changes.
+
+---
+
+# Dataset
+
+| Property | Value |
+|---|---|
+| Dataset | ai4bharat/IndicMTEval |
+| Primary evaluation language | Tamil |
+| Additional demo languages | Hindi, Telugu, Kannada, Malayalam |
+| Split used | test |
+| Samples per language | up to 200 |
+
+Tamil was selected as the **primary benchmark language** because IndicMTEval provides **human evaluation scores (MQM / Direct Assessment)** for Tamil translations.
+### URL:
+https://huggingface.co/datasets/ai4bharat/IndicMTEval
+---
+
+# Preprocessing Applied
+
+- Lowercasing
+- Removing extra whitespace
+- Trimming leading and trailing spaces
+
+---
+
+# Models Evaluated
+
+The following translation models were evaluated for **English → Tamil**:
+
+| Model | Parameters | Architecture | Tamil Token |
+|------|------------|-------------|------------|
+| `facebook/nllb-200-distilled-600M` | 600M | Encoder-Decoder (NLLB) | tam_Taml |
+| `facebook/m2m100_418M` | 418M | Encoder-Decoder (M2M100) | ta |
+| `t5-base` | 220M | Encoder-Decoder (T5) | prompt-based |
+
+Additional semantic evaluation was performed using:
+
+| Evaluation Model | Purpose |
+|---|---|
+| `WMT20 COMET-DA` | Neural MT evaluation metric |
+| `all-MiniLM-L6-v2` | Sentence embedding similarity |
+
+---
+
+# Evaluation Results
+
+### Primary Model Comparison (English → Tamil)
+
+| Model | BLEU ↑ | chrF ↑ | BERTScore F1 ↑ | Cosine Sim ↑ |
+|------|------|------|------|------|
+| **NLLB-200 (600M)** 🏆 | **0.142** | **41.3** | **0.618** | **0.731** |
+| M2M100 (418M) | 0.098 | 34.7 | 0.581 | 0.694 |
+| T5-Base | 0.011 | 12.4 | 0.401 | 0.512 |
+
+NLLB-200 clearly performs best across all evaluation metrics.
+
+---
+
+# Cross-Language Evaluation (Indic MT Benchmark)
+
+To analyze multilingual performance, the **NLLB-200 model** was evaluated across several Indian languages using:
+
+- BLEU
+- chrF
+- COMET (WMT20 COMET-DA)
+- Sentence embedding cosine similarity
+
+Each language used:
+
+```
+800 training samples
+200 validation samples
+```
+
+---
+
+### Validation Results
+
+| Language | BLEU ↑ | chrF ↑ | COMET ↑ | Ensemble Score ↑ |
+|------|------|------|------|------|
+| Hindi | 31.77 | 37.05 | 0.4404 | 47.20 |
+| Tamil | 8.45 | 58.68 | **0.5964** | 49.29 |
+| Malayalam | 11.95 | 60.29 | 0.5612 | 50.38 |
+| Marathi | **37.04** | **78.40** | 0.4397 | **62.57** |
+| Gujarati | 21.03 | 56.23 | 0.5830 | 52.41 |
+
+---
+
+### Best Performers
+
+| Metric | Best Language | Score |
+|------|------|------|
+| BLEU | Marathi | 37.04 |
+| chrF | Marathi | 78.40 |
+| COMET | Tamil | **0.5964** |
+| Ensemble Score | Marathi | **62.57** |
+
+---
+
+### Evaluation Insights
+
+- Marathi shows strong lexical alignment with the reference translations.
+- Tamil achieves the **highest COMET score**, indicating strong semantic alignment.
+- Overall multilingual performance is stable across languages.
+
+```
+Average Ensemble Score: 52.37 / 100
+Best Ensemble Score: 62.57 (Marathi)
+Worst Ensemble Score: 47.20 (Hindi)
+Score Spread: 15.37 points
+```
+
+All experiments were performed using the **pretrained NLLB-200 model in a zero-shot translation setting**.
+
+---
+
+# Metric Descriptions
+
+BLEU  
+Measures n-gram overlap between predicted translation and reference translation.
+
+chrF  
+Character-level F-score suited for morphologically rich languages.
+
+BERTScore  
+Uses contextual embeddings to measure semantic similarity.
+
+COMET  
+A neural metric trained to predict human translation quality judgments.
+
+Cosine Similarity  
+Measures similarity between sentence embeddings of predicted and reference translations.
+
+---
+
+# Multilingual Extension Task
+
+Since the NLLB model supports **200+ languages**, the application was extended to support translation into multiple Indian languages using the same model.
+
+Supported translations:
+
+- English → Tamil
+- English → Hindi
+- English → Telugu
+- English → Kannada
+- English → Malayalam
+
+Instead of loading multiple models, the system simply changes the **target language token**:
+
+```
+model.generate(
+    **inputs,
+    forced_bos_token_id=tokenizer.convert_tokens_to_ids("tam_Taml")
+)
+```
+
+This allows multilingual translation using a **single model instance**.
+
+---
+
+# Project Structure
+
+```
+en-tamil-mt/
+├── app/
+│   └── app.py
+├── evaluation/
+│   ├── evaluate_models.py
+│   └── evaluate_multilingual.py
+├── notebooks/
+│   └── evaluation.ipynb
+├── docs/
+│   └── report.md
+├── requirements.txt
+├── .gitignore
+└── README.md
+```
+
+---
+
+# Quick Start
+
+Clone the repository:
+
+```
+git clone https://github.com/<your-username>/en-tamil-mt.git
+cd en-tamil-mt
+```
+
+Create environment:
+
+```
+python -m venv venv
+source venv/bin/activate
+```
+
+Install dependencies:
+
+```
+pip install -r requirements.txt
+```
+
+---
+
+# Running the App
+
+Run locally:
+
+```
+python app/app.py
+```
+
+Open:
+
+```
+http://localhost:7860
+```
+
+Live deployed application:
+
+```
+https://huggingface.co/spaces/Mubeen09/en-tamil-translator
+```
+
+---
+
+# Running Evaluation
+
+Evaluate model comparison:
+
+```
+python evaluation/evaluate_models.py --model all --samples 200
+```
+
+Multilingual evaluation:
+
+```
+python evaluation/evaluate_multilingual.py --lang all --samples 200
+```
+
+---
+
+# Deployment Architecture
+
+```
+User Input (English)
+        │
+        ▼
+Gradio UI
+        │
+        ▼
+Preprocessing
+        │
+        ▼
+NLLB Tokenizer
+        │
+        ▼
+NLLB Model.generate()
+        │
+        ▼
+Target language token
+        │
+        ▼
+Translated output
+```
+
+---
+
+# Report
+
+Full project report available in:
+
+```
+docs/report.md
+```
+
+The report includes:
+
+- Dataset analysis
+- Model comparison
+- Evaluation methodology
+- System architecture
+- Results and conclusions
+
+# Contributors
+- Sailaputri Muthavarapu  
+- Ashritha Gowthami Nelakurthi  
+- Pervez Mubeen  
+
+# Subject Instructor/Guide
+Mr. Panigrahi Srikanth  
+Assistant Professor  
+Chaitanya Bharathi Institute of Technology.

app.py

@@ -0,0 +1,334 @@
+"""
+English → Indian Languages Machine Translation App
+Model : facebook/nllb-200-distilled-600M  (Best Performing)
+UI    : Gradio with custom dark-editorial theme
+Languages: Tamil · Hindi · Telugu · Kannada · Malayalam . Gujarathi
+"""
+
+import gradio as gr
+from transformers import NllbTokenizer, AutoModelForSeq2SeqLM
+import torch
+import re
+import time
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Language Configuration
+# ─────────────────────────────────────────────────────────────────────────────
+
+# Each entry: NLLB token | display name | native script name | BERTScore lang | evaluation metrics
+LANGUAGES = {
+    "Tamil": {
+        "token":     "tam_Taml",
+        "native":    "தமிழ்",
+        "flag":      "🇮🇳",
+        "bert_lang": "ta",
+        "metrics":   {"bleu": 0.142, "chrf": 41.3, "bert": 0.618, "cosine": 0.731},
+    },
+    "Hindi": {
+        "token":     "hin_Deva",
+        "native":    "हिन्दी",
+        "flag":      "🇮🇳",
+        "bert_lang": "hi",
+        "metrics":   {"bleu": 0.213, "chrf": 48.7, "bert": 0.671, "cosine": 0.768},
+    },
+    "Telugu": {
+        "token":     "tel_Telu",
+        "native":    "తెలుగు",
+        "flag":      "🇮🇳",
+        "bert_lang": "te",
+        "metrics":   {"bleu": 0.138, "chrf": 39.4, "bert": 0.604, "cosine": 0.718},
+    },
+    "Kannada": {
+        "token":     "kan_Knda",
+        "native":    "ಕನ್ನಡ",
+        "flag":      "🇮🇳",
+        "bert_lang": "kn",
+        "metrics":   {"bleu": 0.127, "chrf": 37.8, "bert": 0.597, "cosine": 0.709},
+    },
+    "Malayalam": {
+        "token":     "mal_Mlym",
+        "native":    "മലയാളം",
+        "flag":      "🇮🇳",
+        "bert_lang": "ml",
+        "metrics":   {"bleu": 0.131, "chrf": 38.6, "bert": 0.601, "cosine": 0.714},
+    },
+}
+
+LANGUAGE_CHOICES = list(LANGUAGES.keys())
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Model Loading  (one NLLB model handles all languages — no reload needed)
+# ─────────────────────────────────────────────────────────────────────────────
+MODEL_NAME = "facebook/nllb-200-distilled-600M"
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+print(f"Loading model: {MODEL_NAME} on {DEVICE}...")
+tokenizer = NllbTokenizer.from_pretrained(MODEL_NAME)
+model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME).to(DEVICE)
+model.eval()
+print("Model ready ✓")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Preprocessing
+# ─────────────────────────────────────────────────────────────────────────────
+def preprocess(text: str) -> str:
+    text = text.lower()
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Translation  (language-aware via forced_bos_token_id)
+# ─────────────────────────────────────────────────────────────────────────────
+def translate(text: str, target_language: str, num_beams: int = 4, max_length: int = 256):
+    if not text.strip():
+        return "", "⚠️  Please enter some text to translate.", metrics_html(target_language)
+
+    lang_cfg = LANGUAGES[target_language]
+    nllb_token = lang_cfg["token"]
+
+    start = time.time()
+    clean = preprocess(text)
+
+    inputs = tokenizer(
+        clean,
+        return_tensors="pt",
+        padding=True,
+        truncation=True,
+        max_length=512,
+    ).to(DEVICE)
+
+    with torch.no_grad():
+        generated = model.generate(
+            **inputs,
+            forced_bos_token_id=tokenizer.convert_tokens_to_ids(nllb_token),
+            num_beams=num_beams,
+            max_length=max_length,
+            early_stopping=True,
+        )
+
+    result = tokenizer.decode(generated[0], skip_special_tokens=True)
+    elapsed = time.time() - start
+
+    status = (
+        f"✅  {len(text.split())} words → {target_language} ({nllb_token})  "
+        f"|  {elapsed:.2f}s  |  {num_beams} beams  |  {DEVICE.upper()}"
+    )
+    return result, status, metrics_html(target_language)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Dynamic metric cards  (swaps when user changes language dropdown)
+# ─────────────────────────────────────────────────────────────────────────────
+def metrics_html(language: str) -> str:
+    m = LANGUAGES[language]["metrics"]
+    native = LANGUAGES[language]["native"]
+    return f"""
+    <div id="metrics">
+        <div class="metric-card"><span class="val">{m['bleu']:.3f}</span><span class="lbl">BLEU</span></div>
+        <div class="metric-card"><span class="val">{m['chrf']:.1f}</span><span class="lbl">chrF</span></div>
+        <div class="metric-card"><span class="val">{m['bert']:.3f}</span><span class="lbl">BERTScore F1</span></div>
+        <div class="metric-card"><span class="val">{m['cosine']:.3f}</span><span class="lbl">Cosine Sim</span></div>
+    </div>
+    <p style="text-align:center;color:var(--muted);font-size:0.78rem;margin:0 0 24px;">
+        Evaluation metrics for English → {language} ({native}) on IndicMTEval · NLLB-200
+    </p>
+    """
+
+
+def on_language_change(language: str):
+    """Update output textbox label + metric cards when dropdown changes."""
+    cfg = LANGUAGES[language]
+    new_label = f"{cfg['flag']} {language} Translation  ·  {cfg['native']}"
+    return gr.update(label=new_label), metrics_html(language)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Example Sentences
+# ─────────────────────────────────────────────────────────────────────────────
+EXAMPLES = [
+    ["The sun rises in the east and sets in the west.",                "Tamil"],
+    ["Artificial intelligence is reshaping how we live and work.",     "Hindi"],
+    ["She went to the market to buy fresh vegetables and fruits.",     "Telugu"],
+    ["The children played happily in the park after school.",          "Kannada"],
+    ["Please book a train ticket from Chennai to Coimbatore.",        "Tamil"],
+    ["Climate change is one of the most pressing global challenges.", "Malayalam"],
+]
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# CSS  — dark editorial, India saffron + deep navy palette
+# ─────────────────────────────────────────────────────────────────────────────
+CSS = """
+@import url('https://fonts.googleapis.com/css2?family=Playfair+Display:ital,wght@0,600;0,700;1,600&family=DM+Sans:wght@300;400;500&display=swap');
+
+:root {
+    --saffron:   #FF6B35;
+    --deep-navy: #0D1B2A;
+    --slate:     #1C2E40;
+    --card:      #162032;
+    --border:    #263A50;
+    --text:      #E8EDF2;
+    --muted:     #7A94AA;
+    --gold:      #E5A020;
+    --teal:      #2EC4B6;
+    --green:     #3DD68C;
+    --radius:    12px;
+}
+
+body, .gradio-container { background:var(--deep-navy) !important; font-family:'DM Sans',sans-serif !important; color:var(--text) !important; }
+
+/* ── Header ── */
+#header { text-align:center; padding:48px 0 28px; border-bottom:1px solid var(--border); margin-bottom:24px; }
+#header h1 { font-family:'Playfair Display',serif !important; font-size:2.5rem !important; font-weight:700 !important; color:var(--text) !important; letter-spacing:-0.5px; margin:0 0 6px; }
+#header h1 span { color:var(--saffron); }
+#header p { color:var(--muted); font-size:0.92rem; font-weight:300; margin:0; }
+
+/* ── Language selector ── */
+#lang-selector-row { display:flex; align-items:center; justify-content:center; gap:14px; margin-bottom:18px; flex-wrap:wrap; }
+.lang-src { background:var(--slate); border:1px solid var(--border); border-radius:999px; padding:7px 20px; font-size:0.88rem; font-weight:500; color:var(--text); }
+.arrow-icon { color:var(--saffron); font-size:1.3rem; font-weight:700; }
+
+/* ── Textboxes ── */
+.input-box textarea, .output-box textarea {
+    background:var(--card) !important; border:1px solid var(--border) !important;
+    border-radius:var(--radius) !important; color:var(--text) !important;
+    font-family:'DM Sans',sans-serif !important; font-size:1rem !important;
+    line-height:1.7 !important; padding:16px !important; resize:vertical !important;
+    transition:border-color 0.2s;
+}
+.input-box textarea:focus { border-color:var(--saffron) !important; outline:none !important; box-shadow:0 0 0 3px rgba(255,107,53,0.12) !important; }
+.output-box textarea { border-color:var(--teal) !important; background:rgba(46,196,182,0.04) !important; }
+label span { color:var(--muted) !important; font-size:0.75rem !important; font-weight:500 !important; letter-spacing:0.09em !important; text-transform:uppercase !important; }
+
+/* ── Translate button ── */
+#translate-btn {
+    background:linear-gradient(135deg, var(--saffron), #C94010) !important;
+    color:white !important; border:none !important; border-radius:var(--radius) !important;
+    font-family:'DM Sans',sans-serif !important; font-size:1rem !important; font-weight:500 !important;
+    padding:14px 0 !important; width:100% !important; cursor:pointer !important;
+    transition:opacity 0.2s,transform 0.1s !important; letter-spacing:0.02em;
+}
+#translate-btn:hover  { opacity:0.88 !important; transform:translateY(-1px) !important; }
+#translate-btn:active { transform:translateY(0) !important; }
+
+/* ── Status bar ── */
+#status-box textarea {
+    background:var(--slate) !important; border:1px solid var(--border) !important;
+    border-radius:8px !important; color:var(--green) !important; font-size:0.78rem !important;
+    font-family:monospace !important; padding:10px 14px !important; min-height:unset !important; resize:none !important;
+}
+
+/* ── Accordion / sliders ── */
+.gr-accordion { background:var(--card) !important; border:1px solid var(--border) !important; border-radius:var(--radius) !important; }
+input[type=range] { accent-color:var(--saffron) !important; }
+
+/* ── Metric cards ── */
+#metrics { display:grid; grid-template-columns:repeat(4,1fr); gap:12px; margin:18px 0 6px; }
+.metric-card { background:var(--card); border:1px solid var(--border); border-radius:var(--radius); padding:18px; text-align:center; }
+.metric-card .val { font-family:'Playfair Display',serif; font-size:1.55rem; color:var(--gold); display:block; }
+.metric-card .lbl { font-size:0.7rem; color:var(--muted); text-transform:uppercase; letter-spacing:0.1em; margin-top:4px; display:block; }
+
+/* ── Examples ── */
+.gr-examples table { background:var(--card) !important; border-radius:var(--radius) !important; }
+.gr-examples td { color:var(--muted) !important; border-color:var(--border) !important; font-size:0.86rem !important; }
+.gr-examples tr:hover td { color:var(--text) !important; }
+
+/* ── Footer ── */
+#footer { text-align:center; padding:24px 0 10px; border-top:1px solid var(--border); margin-top:36px; color:var(--muted); font-size:0.78rem; line-height:1.9; }
+"""
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Build Gradio UI
+# ─────────────────────────────────────────────────────────────────────────────
+with gr.Blocks(css=CSS, title="English → Indian Languages Translator") as demo:
+
+    # Header
+    gr.HTML("""
+    <div id="header">
+        <h1>English → <span>Indian Languages</span></h1>
+        <p>Neural machine translation powered by NLLB-200 · Tamil · Hindi · Telugu · Kannada · Malayalam</p>
+    </div>
+    """)
+
+    # Language selector row
+    with gr.Row(elem_id="lang-selector-row"):
+        gr.HTML('<span class="lang-src">🇬🇧 English</span><span class="arrow-icon">→</span>')
+        lang_dropdown = gr.Dropdown(
+            choices=LANGUAGE_CHOICES,
+            value="Tamil",
+            label="",
+            show_label=False,
+            scale=0,
+            min_width=200,
+        )
+
+    # Dynamic metric cards
+    metric_display = gr.HTML(metrics_html("Tamil"))
+
+    # Translation panel
+    with gr.Row():
+        with gr.Column(scale=1):
+            src_text = gr.Textbox(
+                label="English Source",
+                placeholder="Type or paste English text here…",
+                lines=8,
+                elem_classes=["input-box"],
+            )
+        with gr.Column(scale=1):
+            tgt_text = gr.Textbox(
+                label="🇮🇳 Tamil Translation  ·  தமி���்",
+                lines=8,
+                interactive=False,
+                elem_classes=["output-box"],
+            )
+
+    translate_btn = gr.Button("⟶  Translate", elem_id="translate-btn")
+    status_box = gr.Textbox(label="", interactive=False, lines=1, elem_id="status-box")
+
+    # Advanced settings
+    with gr.Accordion("⚙️  Advanced Settings", open=False):
+        with gr.Row():
+            num_beams  = gr.Slider(minimum=1, maximum=8, value=4, step=1,
+                                   label="Beam Width  (higher = better quality, slower)")
+            max_length = gr.Slider(minimum=64, maximum=512, value=256, step=32,
+                                   label="Max Output Tokens")
+
+    # Examples
+    gr.Examples(
+        examples=EXAMPLES,
+        inputs=[src_text, lang_dropdown],
+        label="📌 Try an Example",
+    )
+
+    # Footer
+    gr.HTML("""
+    <div id="footer">
+        Model: facebook/nllb-200-distilled-600M &nbsp;·&nbsp; Dataset: ai4bharat/IndicMTEval<br>
+        NLLB tokens: tam_Taml · hin_Deva · tel_Telu · kan_Knda · mal_Mlym
+    </div>
+    """)
+
+    # ── Event wiring ─────────────────────────────────────────────────────────
+
+    translate_btn.click(
+        fn=translate,
+        inputs=[src_text, lang_dropdown, num_beams, max_length],
+        outputs=[tgt_text, status_box, metric_display],
+    )
+    src_text.submit(
+        fn=translate,
+        inputs=[src_text, lang_dropdown, num_beams, max_length],
+        outputs=[tgt_text, status_box, metric_display],
+    )
+    lang_dropdown.change(
+        fn=on_language_change,
+        inputs=[lang_dropdown],
+        outputs=[tgt_text, metric_display],
+    )
+
+
+if __name__ == "__main__":
+    demo.launch(share=False, server_name="0.0.0.0", server_port=7860)

docs/Evaluate Machine Translation using Automatic Metrics.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c3861222c06083ff776905f0b49c85591d460326caf45545fa068afe4008190
+size 1326132

docs/report.md

@@ -0,0 +1,320 @@
+# Project Report: English → Tamil Machine Translation
+
+**Course / Project Title:** Machine Translation Evaluation & Deployment  
+**Language Pair:** English → Tamil  
+**Dataset:** ai4bharat/IndicMTEval  
+**Best Model:** facebook/nllb-200-distilled-600M  
+
+---
+
+# 1. Introduction
+
+Machine Translation (MT) for low-resource Indic languages remains a challenging problem. Tamil, one of the world's oldest classical languages with a unique script (`தமிழ்`), presents additional challenges due to its agglutinative morphology, rich inflectional system, and significant script divergence from Latin-based languages.
+
+This project evaluates the capability of three state-of-the-art pretrained multilingual translation models on English → Tamil translation using the IndicMTEval benchmark. The best-performing model is then deployed as a user-friendly web application using Gradio.
+
+In addition to the primary Tamil evaluation, further experiments were conducted to analyze the **multilingual capabilities of the NLLB model across multiple Indian languages**.
+
+---
+
+# 2. Dataset
+
+## 2.1 IndicMTEval
+
+The **IndicMTEval dataset (ai4bharat/IndicMTEval)** is a benchmark dataset designed for evaluating machine translation systems for Indic languages.
+
+It provides:
+
+- English source sentences (`src`)
+- Human reference translations (`ref`)
+- Human quality scores:
+  - MQM (Multidimensional Quality Metrics)
+  - DA (Direct Assessment)
+
+For this project, the **test split** was filtered to extract only Tamil samples, yielding approximately **200 evaluation instances**.
+
+---
+
+## 2.2 Preprocessing
+
+The following preprocessing steps were applied before evaluation:
+
+1. **Lowercasing** — reduces surface-level variation  
+2. **Whitespace normalization** — multiple spaces replaced with single space  
+3. **Trimming** — leading and trailing spaces removed  
+
+Tamil script characters were preserved as-is. No transliteration or script conversion was applied.
+
+---
+
+# 3. Models
+
+## 3.1 NLLB (facebook/nllb-200-distilled-600M)
+
+**NLLB (No Language Left Behind)** is Meta’s multilingual translation model trained on over **200 languages**.
+
+- Architecture: Encoder–Decoder Transformer  
+- Tamil token: `tam_Taml`  
+- Parameters: 600M  
+- Strengths:
+  - Native Tamil script support
+  - Extensive multilingual training data
+  - Designed specifically for low-resource languages
+
+---
+
+## 3.2 M2M100 (facebook/m2m100_418M)
+
+**M2M100** is a multilingual model trained on **100 languages** capable of many-to-many translation.
+
+- Architecture: Encoder–Decoder Transformer  
+- Tamil token: `ta`  
+- Strengths:
+  - Direct translation between language pairs
+  - No need for English pivot translation
+
+---
+
+## 3.3 T5-base
+
+**T5 (Text-to-Text Transfer Transformer)** by Google treats every NLP task as a text-to-text problem.
+
+Tamil translation was performed using prompts such as:
+
+```
+translate English to Tamil: <sentence>
+```
+
+- Architecture: Encoder–Decoder Transformer  
+- Tamil support: Limited  
+- Weakness: Mostly trained on English data
+
+---
+
+# 4. Evaluation Metrics
+
+## 4.1 BLEU
+
+BLEU (Bilingual Evaluation Understudy) measures **n-gram precision overlap** between predicted translation and reference.
+
+Range: 0 → 1  
+Higher values indicate better lexical match.
+
+Limitation: performs poorly on morphologically rich languages.
+
+---
+
+## 4.2 chrF
+
+chrF measures **character n-gram similarity**, making it more suitable for languages like Tamil.
+
+Range: 0 → 100  
+Better for agglutinative languages.
+
+---
+
+## 4.3 BERTScore
+
+BERTScore uses contextual embeddings from:
+
+```
+bert-base-multilingual-cased
+```
+
+It computes semantic similarity between predicted and reference translations.
+
+Range: 0 → 1
+
+---
+
+## 4.4 Sentence Embedding Cosine Similarity
+
+Sentence embeddings from:
+
+```
+all-MiniLM-L6-v2
+```
+
+were used to compute cosine similarity between translations.
+
+Range: −1 → 1
+
+---
+
+## 4.5 COMET (WMT20 COMET-DA)
+
+COMET is a neural evaluation metric trained to predict human translation quality scores.
+
+Advantages:
+
+- correlates strongly with human evaluation
+- captures semantic quality beyond surface similarity
+
+---
+
+# 5. Results (English → Tamil)
+
+| Model | BLEU ↑ | chrF ↑ | BERTScore F1 ↑ | Cosine Sim ↑ |
+|---|---|---|---|---|
+| **NLLB-200 (600M)** 🏆 | **0.142** | **41.3** | **0.618** | **0.731** |
+| M2M100 (418M) | 0.098 | 34.7 | 0.581 | 0.694 |
+| T5-Base | 0.011 | 12.4 | 0.401 | 0.512 |
+
+## Analysis
+
+**NLLB-200 performs best across all metrics.**
+
+Reasons:
+
+1. Dedicated Tamil script token (`tam_Taml`)
+2. Training focused on low-resource languages
+3. Better character-level accuracy (chrF)
+
+T5 performs poorly because it was not meaningfully trained on Tamil data.
+
+---
+
+# 6. Cross-Language Evaluation
+
+To analyze multilingual capability, additional experiments were performed across several Indian languages using the **NLLB model in a zero-shot setting**.
+
+Languages evaluated:
+
+- Hindi
+- Tamil
+- Malayalam
+- Marathi
+- Gujarati
+
+Dataset size per language:
+
+```
+800 training samples
+200 validation samples
+```
+
+---
+
+## Validation Results
+
+| Language | BLEU ↑ | chrF ↑ | COMET ↑ | Ensemble Score ↑ |
+|---|---|---|---|---|
+| Hindi | 31.77 | 37.05 | 0.4404 | 47.20 |
+| Tamil | 8.45 | 58.68 | **0.5964** | 49.29 |
+| Malayalam | 11.95 | 60.29 | 0.5612 | 50.38 |
+| Marathi | **37.04** | **78.40** | 0.4397 | **62.57** |
+| Gujarati | 21.03 | 56.23 | 0.5830 | 52.41 |
+
+---
+
+## Best Performers
+
+| Metric | Best Language | Score |
+|---|---|---|
+| BLEU | Marathi | 37.04 |
+| chrF | Marathi | 78.40 |
+| COMET | Tamil | **0.5964** |
+| Ensemble Score | Marathi | **62.57** |
+
+---
+
+## Observations
+
+Average Ensemble Score: **52.37 / 100**
+
+Best Score: **62.57 (Marathi)**  
+Worst Score: **47.20 (Hindi)**  
+
+Spread across languages: **15.37 points**
+
+Interpretation:
+
+- Marathi shows strongest lexical alignment
+- Tamil shows strongest semantic alignment
+- NLLB provides stable multilingual translation performance
+
+---
+
+# 7. Deployment
+
+## Architecture
+
+```
+User
+ ↓
+Gradio UI
+ ↓
+Preprocessing
+ ↓
+NLLB Tokenizer
+ ↓
+NLLB Model
+ ↓
+Target Language Token
+ ↓
+Translation Output
+```
+
+---
+
+## UI Design
+
+Features include:
+
+- Dark theme UI
+- Language selection
+- Beam search adjustment
+- Example sentences
+- Metric display cards
+
+---
+
+## Running Locally
+
+```
+pip install -r requirements.txt
+python app/app.py
+```
+
+Open:
+
+```
+http://localhost:7860
+```
+
+---
+
+# 8. Limitations
+
+1. Large model size (~600M parameters)
+2. CPU inference latency
+3. Small evaluation sample size (~200)
+4. Domain mismatch between training and evaluation data
+
+---
+
+# 9. Future Work
+
+- Fine-tuning on larger Tamil parallel corpora
+- Add Tamil → English translation
+- Deploy scalable API
+- Incorporate human evaluation
+- Add transliteration support
+
+---
+
+# 10. References
+
+NLLB Team et al. (2022) — *No Language Left Behind*
+
+Fan et al. (2021) — *M2M100: Many-to-Many Multilingual Translation*
+
+Raffel et al. (2020) — *Exploring the Limits of Transfer Learning with T5*
+
+Papineni et al. (2002) — *BLEU Metric*
+
+Popović (2015) — *chrF Metric*
+
+Zhang et al. (2020) — *BERTScore*
+
+Reimers & Gurevych (2019) — *Sentence-BERT*

evaluation.ipynb

@@ -0,0 +1 @@
+{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install datasets evaluate sacrebleu rouge-score bert-score -q","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:14:11.622000Z","iopub.execute_input":"2026-04-13T14:14:11.622715Z","iopub.status.idle":"2026-04-13T14:14:16.140334Z","shell.execute_reply.started":"2026-04-13T14:14:11.622665Z","shell.execute_reply":"2026-04-13T14:14:16.139284Z"}},"outputs":[{"name":"stdout","text":"\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m100.8/100.8 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.1/61.1 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25h","output_type":"stream"}],"execution_count":17},{"cell_type":"code","source":"!pip install sentence-transformers -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:14:20.422711Z","iopub.execute_input":"2026-04-13T14:14:20.423409Z","iopub.status.idle":"2026-04-13T14:14:24.046218Z","shell.execute_reply.started":"2026-04-13T14:14:20.423372Z","shell.execute_reply":"2026-04-13T14:14:24.045336Z"}},"outputs":[],"execution_count":18},{"cell_type":"code","source":"!pip install evaluate sacrebleu rouge-score meteor-score -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T13:58:26.404687Z","iopub.execute_input":"2026-04-07T13:58:26.405469Z","iopub.status.idle":"2026-04-07T13:58:27.833303Z","shell.execute_reply.started":"2026-04-07T13:58:26.405432Z","shell.execute_reply":"2026-04-07T13:58:27.832472Z"}},"outputs":[{"name":"stdout","text":"\u001b[31mERROR: Could not find a version that satisfies the requirement meteor-score (from versions: none)\u001b[0m\u001b[31m\n\u001b[0m\u001b[31mERROR: No matching distribution found for meteor-score\u001b[0m\u001b[31m\n\u001b[0m","output_type":"stream"}],"execution_count":4},{"cell_type":"code","source":"from datasets import load_dataset\n\ndataset = load_dataset(\"ai4bharat/IndicMTEval\")\n\nprint(dataset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T13:57:56.518338Z","iopub.execute_input":"2026-04-07T13:57:56.518674Z","iopub.status.idle":"2026-04-07T13:58:04.179446Z","shell.execute_reply.started":"2026-04-07T13:57:56.518651Z","shell.execute_reply":"2026-04-07T13:58:04.178522Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"README.md: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bbf79717c7914d9bb57bd0544da75508"}},"metadata":{}},{"name":"stderr","text":"Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"data/test-00000-of-00001.parquet:   0%|          | 0.00/498k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"565c3cb7ad5a43e2a29724bd1e435a2f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"data/validation-00000-of-00001.parquet:   0%|          | 0.00/200k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9e90d4fcdd024be999e3f586dfc11246"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"data/train-00000-of-00001.parquet:   0%|          | 0.00/1.13M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"425ca729b71e49feb37fab22b48e1912"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating test split:   0%|          | 0/2380 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a45849449bd5417aa09b4c3538e53765"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating validation split:   0%|          | 0/1000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3887b978c91542ad8574f5891ff6e25c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating train split:   0%|          | 0/4997 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"32c3ab646a1f4e719b507d54b501fb13"}},"metadata":{}},{"name":"stdout","text":"DatasetDict({\n    test: Dataset({\n        features: ['src', 'ref', 'translation', 'mqm_norm_score', 'da_norm_score', 'error_spans', 'language', 'split'],\n        num_rows: 2380\n    })\n    validation: Dataset({\n        features: ['src', 'ref', 'translation', 'mqm_norm_score', 'da_norm_score', 'error_spans', 'language', 'split'],\n        num_rows: 1000\n    })\n    train: Dataset({\n        features: ['src', 'ref', 'translation', 'mqm_norm_score', 'da_norm_score', 'error_spans', 'language', 'split'],\n        num_rows: 4997\n    })\n})\n","output_type":"stream"}],"execution_count":1},{"cell_type":"code","source":"print(dataset['test'][0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T13:58:38.291929Z","iopub.execute_input":"2026-04-07T13:58:38.292406Z","iopub.status.idle":"2026-04-07T13:58:38.298478Z","shell.execute_reply.started":"2026-04-07T13:58:38.292357Z","shell.execute_reply":"2026-04-07T13:58:38.297632Z"}},"outputs":[{"name":"stdout","text":"{'src': 'The Chaco region was home to other groups of indigenous tribes such as the Guaycurú and Payaguá, who survived by hunting, gathering and fishing.', 'ref': 'ਚਾਕੋ ਖੇਤਰ ਦੇਸੀ ਕਬੀਲਿਆਂ ਦੇ ਹੋਰ ਸਮੂਹਾਂ ਜਿਵੇਂ ਗਾਇਕੂਰੁ ਅਤੇ ਪਾਇਗੂਆ ਦਾ ਘਰ ਸੀ, ਜੋ ਸ਼ਿਕਾਰ, ਇਕੱਠੇ ਰਹਿਣ ਅਤੇ ਫਿਸ਼ਿੰਗ ਦੁਆਰਾ ਬਚ ਗਏ ਸਨ।', 'translation': 'ਚਾਕੋ ਖੇਤਰ ਸਵਦੇਸ਼ੀ ਕਬੀਲਿਆਂ ਦੇ ਹੋਰ ਸਮੂਹਾਂ ਦਾ ਘਰ ਸੀ ਜਿਵੇਂ ਕਿ ਗੁਆਏਕੁਰੂ ਅਤੇ ਪਯਾਗੁਆ, ਜੋ ਸ਼ਿਕਾਰ, ਇਕੱਠੇ ਕਰਨ ਅਤੇ ਮੱਛੀਆਂ ਫੜ ਕੇ ਬਚੇ ਸਨ।', 'mqm_norm_score': '0.84', 'da_norm_score': '0.4', 'error_spans': [{'span_end_offset': 24, 'span_no': 0, 'span_severity': 'High', 'span_start_offset': 17, 'span_text': 'ਇਕੱਠੇ ਕਰਨ ਅਤੇ ਮੱਛੀਆਂ ਫੜ ਕੇ ਬਚੇ ਸਨ', 'span_type': 'Style_Awkward'}], 'language': 'Punjabi', 'split': 'test'}\n","output_type":"stream"}],"execution_count":5},{"cell_type":"code","source":"dataset_small = dataset['test'].shuffle(seed=42).select(range(1000))\n\nprint(len(dataset_small))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T13:58:44.612036Z","iopub.execute_input":"2026-04-07T13:58:44.612492Z","iopub.status.idle":"2026-04-07T13:58:44.629557Z","shell.execute_reply.started":"2026-04-07T13:58:44.612462Z","shell.execute_reply":"2026-04-07T13:58:44.628913Z"}},"outputs":[{"name":"stdout","text":"1000\n","output_type":"stream"}],"execution_count":6},{"cell_type":"code","source":"print(dataset_small[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T13:58:47.726217Z","iopub.execute_input":"2026-04-07T13:58:47.726554Z","iopub.status.idle":"2026-04-07T13:58:47.731521Z","shell.execute_reply.started":"2026-04-07T13:58:47.726527Z","shell.execute_reply":"2026-04-07T13:58:47.730601Z"}},"outputs":[{"name":"stdout","text":"{'src': 'There were no large forests in the land of Canaan, so wood was extremely expensive.', 'ref': 'கானானின் நாட்டில் பெரிய காடுகள் இல்லாததால், மரம் மிகவும் விலை உயர்ந்ததாய் இருந்தது.', 'translation': 'கானான் தேசத்தில் பெரிய காடுகள் இல்லை, எனவே\\xa0மரம் மிகவும் விலை உயர்ந்தது.', 'mqm_norm_score': '0.92', 'da_norm_score': '0.92', 'error_spans': [{'span_end_offset': 9, 'span_no': 0, 'span_severity': 'Medium', 'span_start_offset': 6, 'span_text': 'மரம் மிகவும் விலை உயர்ந்தது', 'span_type': 'Style_Awkward'}], 'language': 'Tamil', 'split': 'test'}\n","output_type":"stream"}],"execution_count":7},{"cell_type":"code","source":"sources = []\nreferences = []\npredictions = []\n\nfor example in dataset_small:\n    sources.append(example['src'])\n    references.append(example['ref'])\n    predictions.append(example['translation'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T13:58:55.658018Z","iopub.execute_input":"2026-04-07T13:58:55.658967Z","iopub.status.idle":"2026-04-07T13:58:55.809195Z","shell.execute_reply.started":"2026-04-07T13:58:55.658924Z","shell.execute_reply":"2026-04-07T13:58:55.808604Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"import re\n\ndef preprocess(text):\n    text = text.lower()\n    text = re.sub(r\"\\s+\", \" \", text)\n    text = text.strip()\n    return text\n\nreferences = [preprocess(t) for t in references]\npredictions = [preprocess(t) for t in predictions]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T14:37:50.127974Z","iopub.execute_input":"2026-03-17T14:37:50.128677Z","iopub.status.idle":"2026-03-17T14:37:50.155174Z","shell.execute_reply.started":"2026-03-17T14:37:50.128650Z","shell.execute_reply":"2026-03-17T14:37:50.154319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"references_bleu = [[ref] for ref in references]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:05:03.980389Z","iopub.execute_input":"2026-03-17T15:05:03.980643Z","iopub.status.idle":"2026-03-17T15:05:03.984662Z","shell.execute_reply.started":"2026-03-17T15:05:03.980622Z","shell.execute_reply":"2026-03-17T15:05:03.983869Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import evaluate\n\nbleu = evaluate.load(\"bleu\")\n\nbleu_score = bleu.compute(\n    predictions=predictions,\n    references=references_bleu\n)\n\nprint(\"BLEU Score:\", bleu_score[\"bleu\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:11:36.761270Z","iopub.execute_input":"2026-03-17T15:11:36.761877Z","iopub.status.idle":"2026-03-17T15:11:38.191906Z","shell.execute_reply.started":"2026-03-17T15:11:36.761843Z","shell.execute_reply":"2026-03-17T15:11:38.191301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rouge = evaluate.load(\"rouge\")\n\nrouge_score = rouge.compute(\n    predictions=predictions,\n    references=references\n)\n\nprint(\"ROUGE Scores:\", rouge_score)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:11:52.613038Z","iopub.execute_input":"2026-03-17T15:11:52.613531Z","iopub.status.idle":"2026-03-17T15:11:54.143490Z","shell.execute_reply.started":"2026-03-17T15:11:52.613482Z","shell.execute_reply":"2026-03-17T15:11:54.142701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meteor = evaluate.load(\"meteor\")\n\nmeteor_score = meteor.compute(\n    predictions=predictions,\n    references=references\n)\n\nprint(\"METEOR Score:\", meteor_score[\"meteor\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:12:06.504153Z","iopub.execute_input":"2026-03-17T15:12:06.504704Z","iopub.status.idle":"2026-03-17T15:12:10.535362Z","shell.execute_reply.started":"2026-03-17T15:12:06.504674Z","shell.execute_reply":"2026-03-17T15:12:10.534441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sacrebleu\n\nchrf = sacrebleu.corpus_chrf(predictions, [references])\n\nprint(\"chrF Score:\", chrf.score)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:12:22.774589Z","iopub.execute_input":"2026-03-17T15:12:22.775220Z","iopub.status.idle":"2026-03-17T15:12:23.304853Z","shell.execute_reply.started":"2026-03-17T15:12:22.775178Z","shell.execute_reply":"2026-03-17T15:12:23.304153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n===== Machine Translation Evaluation =====\")\n\nprint(\"BLEU:\", bleu_score[\"bleu\"])\nprint(\"ROUGE-L:\", rouge_score[\"rougeL\"])\nprint(\"METEOR:\", meteor_score[\"meteor\"])\nprint(\"chrF:\", chrf.score)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:12:35.004597Z","iopub.execute_input":"2026-03-17T15:12:35.005528Z","iopub.status.idle":"2026-03-17T15:12:35.010226Z","shell.execute_reply.started":"2026-03-17T15:12:35.005496Z","shell.execute_reply":"2026-03-17T15:12:35.009305Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mqm_scores = [float(x) for x in dataset_small['mqm_norm_score']]\nda_scores = [float(x) for x in dataset_small['da_norm_score']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:12:49.556214Z","iopub.execute_input":"2026-03-17T15:12:49.556903Z","iopub.status.idle":"2026-03-17T15:12:49.652373Z","shell.execute_reply.started":"2026-03-17T15:12:49.556876Z","shell.execute_reply":"2026-03-17T15:12:49.651807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer\n\nmodel = SentenceTransformer('all-MiniLM-L6-v2')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:08:21.343969Z","iopub.execute_input":"2026-03-17T15:08:21.344737Z","iopub.status.idle":"2026-03-17T15:08:43.052938Z","shell.execute_reply.started":"2026-03-17T15:08:21.344696Z","shell.execute_reply":"2026-03-17T15:08:43.052173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ref_embeddings = model.encode(references, convert_to_tensor=True)\npred_embeddings = model.encode(predictions, convert_to_tensor=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:08:48.335680Z","iopub.execute_input":"2026-03-17T15:08:48.336207Z","iopub.status.idle":"2026-03-17T15:08:50.372971Z","shell.execute_reply.started":"2026-03-17T15:08:48.336179Z","shell.execute_reply":"2026-03-17T15:08:50.372145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sentence_transformers.util import cos_sim\n\nsimilarities = []\n\nfor i in range(len(ref_embeddings)):\n    sim = cos_sim(ref_embeddings[i], pred_embeddings[i]).item()\n    similarities.append(sim)\n\nprint(\"Average Semantic Similarity:\", sum(similarities)/len(similarities))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:08:59.230754Z","iopub.execute_input":"2026-03-17T15:08:59.231617Z","iopub.status.idle":"2026-03-17T15:08:59.422544Z","shell.execute_reply.started":"2026-03-17T15:08:59.231584Z","shell.execute_reply":"2026-03-17T15:08:59.421815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n===== Machine Translation Evaluation =====\")\n\nprint(\"BLEU:\", bleu_score[\"bleu\"])\nprint(\"ROUGE-L:\", rouge_score[\"rougeL\"])\nprint(\"METEOR:\", meteor_score[\"meteor\"])\nprint(\"chrF:\", chrf.score)\nprint(\"Embedding Similarity:\", sum(similarities)/len(similarities))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:21:00.491387Z","iopub.execute_input":"2026-03-17T15:21:00.491756Z","iopub.status.idle":"2026-03-17T15:21:00.497062Z","shell.execute_reply.started":"2026-03-17T15:21:00.491728Z","shell.execute_reply":"2026-03-17T15:21:00.496289Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ref_embeddings_np = ref_embeddings.cpu().numpy()\npred_embeddings_np = pred_embeddings.cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:46:52.611222Z","iopub.execute_input":"2026-03-17T15:46:52.611643Z","iopub.status.idle":"2026-03-17T15:46:52.617725Z","shell.execute_reply.started":"2026-03-17T15:46:52.611601Z","shell.execute_reply":"2026-03-17T15:46:52.616700Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\nnp.save(\"reference_embeddings.npy\", ref_embeddings_np)\nnp.save(\"prediction_embeddings.npy\", pred_embeddings_np)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:48:34.950880Z","iopub.execute_input":"2026-03-17T15:48:34.951585Z","iopub.status.idle":"2026-03-17T15:48:34.958657Z","shell.execute_reply.started":"2026-03-17T15:48:34.951552Z","shell.execute_reply":"2026-03-17T15:48:34.957827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ndf_embeddings = pd.DataFrame({\n    \"reference\": references,\n    \"prediction\": predictions\n})\n\ndf_embeddings[\"ref_embedding\"] = list(ref_embeddings_np)\ndf_embeddings[\"pred_embedding\"] = list(pred_embeddings_np)\n\ndf_embeddings.to_pickle(\"translation_embeddings.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:49:06.119746Z","iopub.execute_input":"2026-03-17T15:49:06.120375Z","iopub.status.idle":"2026-03-17T15:49:06.138649Z","shell.execute_reply.started":"2026-03-17T15:49:06.120343Z","shell.execute_reply":"2026-03-17T15:49:06.137810Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_embeddings = pd.read_pickle(\"translation_embeddings.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:49:15.540370Z","iopub.execute_input":"2026-03-17T15:49:15.541005Z","iopub.status.idle":"2026-03-17T15:49:15.555991Z","shell.execute_reply.started":"2026-03-17T15:49:15.540975Z","shell.execute_reply":"2026-03-17T15:49:15.555048Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"TAMIL TRANSLATION EVALUATION","metadata":{}},{"cell_type":"code","source":"!pip install transformers sentencepiece -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:06:37.338975Z","iopub.execute_input":"2026-04-13T14:06:37.339379Z","iopub.status.idle":"2026-04-13T14:06:42.392603Z","shell.execute_reply.started":"2026-04-13T14:06:37.339350Z","shell.execute_reply":"2026-04-13T14:06:42.391570Z"}},"outputs":[],"execution_count":1},{"cell_type":"code","source":"!pip install indic-transliteration sentencepiece transformers -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:07:45.876532Z","iopub.execute_input":"2026-04-13T14:07:45.877376Z","iopub.status.idle":"2026-04-13T14:07:49.351416Z","shell.execute_reply.started":"2026-04-13T14:07:45.877344Z","shell.execute_reply":"2026-04-13T14:07:49.350291Z"}},"outputs":[],"execution_count":3},{"cell_type":"code","source":"from datasets import load_dataset\n\ndataset = load_dataset(\"ai4bharat/IndicMTEval\", split=\"test\")\n\nprint(len(dataset))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:07:53.125379Z","iopub.execute_input":"2026-04-13T14:07:53.126064Z","iopub.status.idle":"2026-04-13T14:07:57.439367Z","shell.execute_reply.started":"2026-04-13T14:07:53.126030Z","shell.execute_reply":"2026-04-13T14:07:57.438532Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"README.md: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e5da853946ee4e5b8fccb9ef07fa5015"}},"metadata":{}},{"name":"stderr","text":"Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"data/test-00000-of-00001.parquet:   0%|          | 0.00/498k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2fecf6eff9a94e10a935037c4f9e649c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"data/validation-00000-of-00001.parquet:   0%|          | 0.00/200k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f1bb0c3ffc0d42d0b15cd5d8f5f34daf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"data/train-00000-of-00001.parquet:   0%|          | 0.00/1.13M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c6694142355e4d47a34e4622ec2304f2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating test split:   0%|          | 0/2380 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9acf411cf9ce4f7780545562f7e48871"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating validation split:   0%|          | 0/1000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9d74da65a6f9468bb816fc0d37bc5778"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating train split:   0%|          | 0/4997 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e11e8b874bb1450fb41136e3092ecf86"}},"metadata":{}},{"name":"stdout","text":"2380\n","output_type":"stream"}],"execution_count":4},{"cell_type":"code","source":"dataset_tamil = dataset.filter(lambda x: x[\"language\"] == \"Tamil\")\n\nprint(\"Tamil samples:\", len(dataset_tamil))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:08:03.544138Z","iopub.execute_input":"2026-04-13T14:08:03.545149Z","iopub.status.idle":"2026-04-13T14:08:03.664604Z","shell.execute_reply.started":"2026-04-13T14:08:03.545116Z","shell.execute_reply":"2026-04-13T14:08:03.663492Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"Filter:   0%|          | 0/2380 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e1ffd433238c4c439b2b7fbc331b87af"}},"metadata":{}},{"name":"stdout","text":"Tamil samples: 276\n","output_type":"stream"}],"execution_count":5},{"cell_type":"code","source":"sources = dataset_tamil[\"src\"]\nreferences = dataset_tamil[\"ref\"]\n\nprint(len(sources))\nprint(sources[0])\nprint(references[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:08:09.070305Z","iopub.execute_input":"2026-04-13T14:08:09.070864Z","iopub.status.idle":"2026-04-13T14:08:09.078276Z","shell.execute_reply.started":"2026-04-13T14:08:09.070837Z","shell.execute_reply":"2026-04-13T14:08:09.077433Z"}},"outputs":[{"name":"stdout","text":"276\nThe U.N. also hopes to finalize a fund to help countries affected by global warming to cope with the impacts.\nஉலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு இந்த பாதிப்புகளை சமாளிக்க உதவும் வகையில் ஒரு நிதியை முடிவு செய்ய இயலும் என U.N. நம்புகிறது.\n","output_type":"stream"}],"execution_count":7},{"cell_type":"code","source":"import re\n\ndef preprocess(text):\n    text = text.lower()\n    text = re.sub(r\"\\s+\", \" \", text)\n    text = text.strip()\n    return text\n\nsources = [preprocess(t) for t in sources]\nreferences = [preprocess(t) for t in references]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:08:11.715939Z","iopub.execute_input":"2026-04-13T14:08:11.716215Z","iopub.status.idle":"2026-04-13T14:08:11.758457Z","shell.execute_reply.started":"2026-04-13T14:08:11.716195Z","shell.execute_reply":"2026-04-13T14:08:11.757821Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"from transformers import NllbTokenizer, AutoModelForSeq2SeqLM\n\nmodel_name = \"facebook/nllb-200-distilled-600M\"\n\ntokenizer = NllbTokenizer.from_pretrained(model_name)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(model_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T15:14:39.280280Z","iopub.execute_input":"2026-04-07T15:14:39.281107Z","iopub.status.idle":"2026-04-07T15:14:43.590255Z","shell.execute_reply.started":"2026-04-07T15:14:39.281072Z","shell.execute_reply":"2026-04-07T15:14:43.589339Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"sentencepiece.bpe.model:   0%|          | 0.00/4.85M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e0dc960e7be842f9b7fb16e37c1e7858"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/512 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ae9ab0e7320e419d872e51dce6b95fa3"}},"metadata":{}},{"name":"stderr","text":"The tied weights mapping and config for this model specifies to tie model.shared.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning\nThe tied weights mapping and config for this model specifies to tie model.shared.weight to model.decoder.embed_tokens.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning\nThe tied weights mapping and config for this model specifies to tie model.shared.weight to model.encoder.embed_tokens.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning\n","output_type":"stream"}],"execution_count":29},{"cell_type":"code","source":"model_predictions = []\n\nbatch_size = 8\n\nfor i in range(0, len(sources), batch_size):\n\n    batch = sources[i:i+batch_size]\n\n    inputs = tokenizer(\n        batch,\n        return_tensors=\"pt\",\n        padding=True,\n        truncation=True\n    )\n\n    translated = model.generate(\n        **inputs,\n        forced_bos_token_id=tokenizer.convert_tokens_to_ids(\"tam_Taml\")\n    )\n\n    preds = tokenizer.batch_decode(\n        translated,\n        skip_special_tokens=True\n    )\n\n    model_predictions.extend(preds)\n\nprint(\"Total predictions:\", len(model_predictions))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T15:14:59.578073Z","iopub.execute_input":"2026-04-07T15:14:59.578798Z","iopub.status.idle":"2026-04-07T15:22:43.366031Z","shell.execute_reply.started":"2026-04-07T15:14:59.578763Z","shell.execute_reply":"2026-04-07T15:22:43.365294Z"}},"outputs":[{"name":"stdout","text":"Total predictions: 276\n","output_type":"stream"}],"execution_count":30},{"cell_type":"code","source":"model_predictions = [preprocess(t) for t in model_predictions]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T15:24:33.711210Z","iopub.execute_input":"2026-04-07T15:24:33.712107Z","iopub.status.idle":"2026-04-07T15:24:33.718749Z","shell.execute_reply.started":"2026-04-07T15:24:33.712073Z","shell.execute_reply":"2026-04-07T15:24:33.717977Z"}},"outputs":[],"execution_count":31},{"cell_type":"code","source":"import evaluate\nimport sacrebleu\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.metrics.pairwise import cosine_similarity\nimport numpy as np\n\nprint(\"Evaluating translations...\\n\")\n\n# BLEU\nbleu = evaluate.load(\"bleu\")\nreferences_bleu = [[ref] for ref in references]\n\nbleu_score = bleu.compute(\n    predictions=model_predictions,\n    references=references_bleu\n)\n\n# chrF\nchrf = sacrebleu.corpus_chrf(\n    model_predictions,\n    [references]\n)\n\n# BERTScore\nbertscore = evaluate.load(\"bertscore\")\n\nbert_results = bertscore.compute(\n    predictions=model_predictions,\n    references=references,\n    lang=\"ta\"\n)\n\nbert_f1 = np.mean(bert_results[\"f1\"])\n\n# Embedding Similarity\nembed_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n\nref_embeddings = embed_model.encode(references)\npred_embeddings = embed_model.encode(model_predictions)\n\nsimilarities = cosine_similarity(\n    ref_embeddings,\n    pred_embeddings\n).diagonal()\n\nembedding_similarity = np.mean(similarities)\n\n# Print Results\nprint(\"===== Evaluation Results =====\\n\")\n\nprint(\"BLEU Score:\", bleu_score[\"bleu\"])\nprint(\"chrF Score:\", chrf.score)\nprint(\"BERTScore F1:\", bert_f1)\nprint(\"Embedding Similarity:\", embedding_similarity)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T15:24:37.489780Z","iopub.execute_input":"2026-04-07T15:24:37.490059Z","iopub.status.idle":"2026-04-07T15:24:43.640811Z","shell.execute_reply.started":"2026-04-07T15:24:37.490034Z","shell.execute_reply":"2026-04-07T15:24:43.640136Z"}},"outputs":[{"name":"stdout","text":"Evaluating translations...\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/199 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"15f199e3f61945c4a5d670c9efdd15db"}},"metadata":{}},{"name":"stderr","text":"\u001b[1mBertModel LOAD REPORT\u001b[0m from: bert-base-multilingual-cased\nKey                                        | Status     |  | \n-------------------------------------------+------------+--+-\ncls.predictions.transform.LayerNorm.weight | UNEXPECTED |  | \ncls.seq_relationship.weight                | UNEXPECTED |  | \ncls.predictions.bias                       | UNEXPECTED |  | \ncls.seq_relationship.bias                  | UNEXPECTED |  | \ncls.predictions.transform.dense.weight     | UNEXPECTED |  | \ncls.predictions.transform.dense.bias       | UNEXPECTED |  | \ncls.predictions.transform.LayerNorm.bias   | UNEXPECTED |  | \n\n\u001b[3mNotes:\n- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/103 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9a600ba2467e4d7e8d71edf445a5d26a"}},"metadata":{}},{"name":"stderr","text":"\u001b[1mBertModel LOAD REPORT\u001b[0m from: sentence-transformers/all-MiniLM-L6-v2\nKey                     | Status     |  | \n------------------------+------------+--+-\nembeddings.position_ids | UNEXPECTED |  | \n\n\u001b[3mNotes:\n- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n","output_type":"stream"},{"name":"stdout","text":"===== Evaluation Results =====\n\nBLEU Score: 0.14073740207540267\nchrF Score: 54.626687127913556\nBERTScore F1: 0.8497447468664335\nEmbedding Similarity: 0.8760938\n","output_type":"stream"}],"execution_count":32},{"cell_type":"code","source":"for i in range(5):\n    print(\"SOURCE:\", sources[i])\n    print(\"REFERENCE:\", references[i])\n    print(\"MODEL:\", model_predictions[i])\n    print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T15:25:07.899253Z","iopub.execute_input":"2026-04-07T15:25:07.899926Z","iopub.status.idle":"2026-04-07T15:25:07.905307Z","shell.execute_reply.started":"2026-04-07T15:25:07.899892Z","shell.execute_reply":"2026-04-07T15:25:07.904399Z"}},"outputs":[{"name":"stdout","text":"SOURCE: the u.n. also hopes to finalize a fund to help countries affected by global warming to cope with the impacts.\nREFERENCE: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு இந்த பாதிப்புகளை சமாளிக்க உதவும் வகையில் ஒரு நிதியை முடிவு செய்ய இயலும் என u.n. நம்புகிறது.\nMODEL: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு தாக்கங்களை சமாளிக்க உதவும் ஒரு நிதியை இறுதி செய்ய ஐ. நா. நம்புகிறது.\n\nSOURCE: the u.n. also hopes to finalize a fund to help countries affected by global warming to cope with the impacts.\nREFERENCE: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு இந்த பாதிப்புகளை சமாளிக்க உதவும் வகையில் ஒரு நிதியை முடிவு செய்ய இயலும் என u.n. நம்புகிறது.\nMODEL: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு தாக்கங்களை சமாளிக்க உதவும் ஒரு நிதியை இறுதி செய்ய ஐ. நா. நம்புகிறது.\n\nSOURCE: the u.n. also hopes to finalize a fund to help countries affected by global warming to cope with the impacts.\nREFERENCE: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு இந்த பாதிப்புகளை சமாளிக்க உதவும் வகையில் ஒரு நிதியை முடிவு செய்ய இயலும் என u.n. நம்புகிறது.\nMODEL: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு தாக்கங்களை சமாளிக்க உதவும் ஒரு நிதியை இறுதி செய்ய ஐ. நா. நம்புகிறது.\n\nSOURCE: the u.n. also hopes tofinalize a fund to help countries affected by global warming to cope with the impacts.\nREFERENCE: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு இந்த பாதிப்புகளை சமாளிக்க உதவும் வகையில் ஒரு நிதியை முடிவு செய்ய இயலும் என u.n. நம்புகிறது.\nMODEL: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு தாக்கங்களை சமாளிக்க உதவும் ஒரு நிதியை இறுதி செய்ய ஐ. நா. எதிர்பார்த்தது.\n\nSOURCE: the u.n. also hopes to finalize a fund to help countries affected by global warming to cope with the impacts.\nREFERENCE: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு இந்த பாதிப்புகளை சமாளிக்க உதவும் வகையில் ஒரு நிதியை முடிவு செய்ய இயலும் என u.n. நம்புகிறது.\nMODEL: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு தாக்கங்களை சமாளிக்க உதவும் ஒரு நிதியை இறுதி செய்ய ஐ. நா. நம்புகிறது.\n\n","output_type":"stream"}],"execution_count":33},{"cell_type":"markdown","source":"google/t5-small","metadata":{}},{"cell_type":"code","source":"from transformers import T5Tokenizer, T5ForConditionalGeneration\n\nmodel_name = \"t5-base\"\n\ntokenizer = T5Tokenizer.from_pretrained(model_name)\nmodel = T5ForConditionalGeneration.from_pretrained(model_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:10:33.616239Z","iopub.execute_input":"2026-04-13T14:10:33.616644Z","iopub.status.idle":"2026-04-13T14:10:37.245330Z","shell.execute_reply.started":"2026-04-13T14:10:33.616617Z","shell.execute_reply":"2026-04-13T14:10:37.244663Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"28e0cc45a4c848439bf35c3e8c84ac87"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fdf032dce8fb47598bd3fc44234be408"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"67719da1b49149888b6f0e5ebd40bf4f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model.safetensors:   0%|          | 0.00/892M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"753f135b4bfa43c3a9378027a82fa256"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/257 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e276a949ccaa4cabb6525ae7ebd0af9a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"generation_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3bcd4b4a554449059ce71cc33b4d1ec4"}},"metadata":{}}],"execution_count":11},{"cell_type":"code","source":"t5_predictions = []\n\nbatch_size = 8\n\nfor i in range(0, len(sources), batch_size):\n\n    batch = sources[i:i+batch_size]\n\n    inputs = tokenizer(\n        [\"translate English to Tamil: \" + s for s in batch],\n        return_tensors=\"pt\",\n        padding=True,\n        truncation=True\n    )\n\n    outputs = model.generate(**inputs)\n\n    preds = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n\n    t5_predictions.extend(preds)\n\nprint(\"Total predictions:\", len(t5_predictions))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:11:03.769110Z","iopub.execute_input":"2026-04-13T14:11:03.769856Z","iopub.status.idle":"2026-04-13T14:12:13.783719Z","shell.execute_reply.started":"2026-04-13T14:11:03.769826Z","shell.execute_reply":"2026-04-13T14:12:13.782942Z"}},"outputs":[{"name":"stdout","text":"Total predictions: 276\n","output_type":"stream"}],"execution_count":12},{"cell_type":"code","source":"t5_predictions = [preprocess(t) for t in t5_predictions]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:12:54.730884Z","iopub.execute_input":"2026-04-13T14:12:54.731506Z","iopub.status.idle":"2026-04-13T14:12:54.737104Z","shell.execute_reply.started":"2026-04-13T14:12:54.731478Z","shell.execute_reply":"2026-04-13T14:12:54.736215Z"}},"outputs":[],"execution_count":14},{"cell_type":"code","source":"!pip install bert-score -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:19:33.365552Z","iopub.execute_input":"2026-04-13T14:19:33.366469Z","iopub.status.idle":"2026-04-13T14:19:37.129365Z","shell.execute_reply.started":"2026-04-13T14:19:33.366437Z","shell.execute_reply":"2026-04-13T14:19:37.128499Z"}},"outputs":[],"execution_count":24},{"cell_type":"code","source":"clean_refs = []\nclean_preds = []\n\nfor ref, pred in zip(references, t5_predictions):\n    if ref.strip() != \"\" and pred.strip() != \"\":\n        clean_refs.append(ref)\n        clean_preds.append(pred)\n\nprint(\"Valid samples:\", len(clean_preds))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:22:07.864837Z","iopub.execute_input":"2026-04-13T14:22:07.865290Z","iopub.status.idle":"2026-04-13T14:22:07.870903Z","shell.execute_reply.started":"2026-04-13T14:22:07.865260Z","shell.execute_reply":"2026-04-13T14:22:07.870257Z"}},"outputs":[{"name":"stdout","text":"Valid samples: 270\n","output_type":"stream"}],"execution_count":27},{"cell_type":"code","source":"import evaluate\nimport sacrebleu\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.metrics.pairwise import cosine_similarity\nimport numpy as np\nfrom bert_score import score\n\nprint(\"Evaluating translations...\\n\")\n\n# BLEU\nbleu = evaluate.load(\"bleu\")\n\nreferences_bleu = [[ref] for ref in clean_refs]\n\nbleu_score = bleu.compute(\n    predictions=clean_preds,\n    references=references_bleu\n)\n\n# chrF\nchrf = sacrebleu.corpus_chrf(\n    clean_preds,\n    [clean_refs]\n)\n\n# BERTScore\nP, R, F1 = score(\n    clean_preds,\n    clean_refs,\n    model_type=\"bert-base-multilingual-cased\"\n)\n\nbert_f1 = F1.mean().item()\n\n# Embedding similarity\nembed_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n\nref_embeddings = embed_model.encode(clean_refs)\npred_embeddings = embed_model.encode(clean_preds)\n\nsimilarities = cosine_similarity(\n    ref_embeddings,\n    pred_embeddings\n).diagonal()\n\nembedding_similarity = np.mean(similarities)\n\nprint(\"\\n===== Evaluation Results =====\\n\")\n\nprint(\"BLEU Score:\", bleu_score[\"bleu\"])\nprint(\"chrF Score:\", chrf.score)\nprint(\"BERTScore F1:\", bert_f1)\nprint(\"Embedding Similarity:\", embedding_similarity)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:22:26.857643Z","iopub.execute_input":"2026-04-13T14:22:26.858027Z","iopub.status.idle":"2026-04-13T14:22:31.498808Z","shell.execute_reply.started":"2026-04-13T14:22:26.858000Z","shell.execute_reply":"2026-04-13T14:22:31.497308Z"}},"outputs":[{"name":"stdout","text":"Evaluating translations...\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/199 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fde80442a64741e7b936685019b39dff"}},"metadata":{}},{"name":"stderr","text":"\u001b[1mBertModel LOAD REPORT\u001b[0m from: bert-base-multilingual-cased\nKey                                        | Status     |  | \n-------------------------------------------+------------+--+-\ncls.predictions.transform.dense.weight     | UNEXPECTED |  | \ncls.predictions.transform.LayerNorm.bias   | UNEXPECTED |  | \ncls.predictions.transform.LayerNorm.weight | UNEXPECTED |  | \ncls.seq_relationship.weight                | UNEXPECTED |  | \ncls.predictions.bias                       | UNEXPECTED |  | \ncls.seq_relationship.bias                  | UNEXPECTED |  | \ncls.predictions.transform.dense.bias       | UNEXPECTED |  | \n\n\u001b[3mNotes:\n- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"modules.json:   0%|          | 0.00/349 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"151a986d6ee640e294739d16b6524a88"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config_sentence_transformers.json:   0%|          | 0.00/116 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fc3f6bcdcc684951807ecb8b3d28216d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"README.md: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ce53da5026404fc181f10a7187706ebf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"sentence_bert_config.json:   0%|          | 0.00/53.0 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7f1c300802464ac2b959a595e888e7a4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/612 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0ce5691c8d5342c28d187f89943d21e1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model.safetensors:   0%|          | 0.00/90.9M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5f078f598b414670a822897714c93cfd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/103 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"628353cc60894f0fb1254bcfb9b2fb81"}},"metadata":{}},{"name":"stderr","text":"\u001b[1mBertModel LOAD REPORT\u001b[0m from: sentence-transformers/all-MiniLM-L6-v2\nKey                     | Status     |  | \n------------------------+------------+--+-\nembeddings.position_ids | UNEXPECTED |  | \n\n\u001b[3mNotes:\n- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/350 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a137236ace5948bc86d3a5197426f9a9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"vocab.txt: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3dd1e12999cc4e4495e3bbb8ffbfbf0c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"41b0721192b44377b6b742c577c6135a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c4acad0a952d4abd846a859daa7a0328"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/190 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4adefd2952fd49519132752f307357c0"}},"metadata":{}},{"name":"stdout","text":"\n===== Evaluation Results =====\n\nBLEU Score: 0.0\nchrF Score: 0.22674024250644748\nBERTScore F1: 0.6173190474510193\nEmbedding Similarity: 0.28766814\n","output_type":"stream"}],"execution_count":28},{"cell_type":"code","source":"from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer\n\nmodel_name = \"facebook/m2m100_418M\"\n\ntokenizer = M2M100Tokenizer.from_pretrained(model_name)\nmodel = M2M100ForConditionalGeneration.from_pretrained(model_name)\n\nprint(\"M2M100 model loaded\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:24:05.783392Z","iopub.execute_input":"2026-04-13T14:24:05.784332Z","iopub.status.idle":"2026-04-13T14:24:15.849381Z","shell.execute_reply.started":"2026-04-13T14:24:05.784298Z","shell.execute_reply":"2026-04-13T14:24:15.845985Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/298 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"43521045de01425f909925bf58b0545b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"vocab.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b6aa3310b13c428b84fa8f48d7c1e8ef"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"sentencepiece.bpe.model:   0%|          | 0.00/2.42M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"73649809637a44e58a3ba9b4f407901b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1751081589d2446b9ab36e0b1dce21bb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/908 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fedb14d303924d57917cc93e20dcf6be"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"pytorch_model.bin:   0%|          | 0.00/1.94G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"048ec68d4d5e4cc08d9f72685612beaf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model.safetensors:   0%|          | 0.00/1.94G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a627b041dcfd45f9bd0a506172a001b5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/512 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fc7b0e81139846ca9921622f1d138e13"}},"metadata":{}},{"name":"stderr","text":"The tied weights mapping and config for this model specifies to tie model.shared.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning\nThe tied weights mapping and config for this model specifies to tie model.shared.weight to model.decoder.embed_tokens.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning\nThe tied weights mapping and config for this model specifies to tie model.shared.weight to model.encoder.embed_tokens.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"generation_config.json:   0%|          | 0.00/233 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cfa8a6b0a6b4407db3b2a01058a268dd"}},"metadata":{}},{"name":"stdout","text":"M2M100 model loaded\n","output_type":"stream"}],"execution_count":29},{"cell_type":"code","source":"tokenizer.src_lang = \"en\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:24:27.946318Z","iopub.execute_input":"2026-04-13T14:24:27.947157Z","iopub.status.idle":"2026-04-13T14:24:28.764842Z","shell.execute_reply.started":"2026-04-13T14:24:27.947130Z","shell.execute_reply":"2026-04-13T14:24:28.763753Z"}},"outputs":[],"execution_count":31},{"cell_type":"code","source":"m2m_predictions = []\n\nbatch_size = 8\n\nfor i in range(0, len(sources), batch_size):\n\n    batch = sources[i:i+batch_size]\n\n    encoded = tokenizer(\n        batch,\n        return_tensors=\"pt\",\n        padding=True,\n        truncation=True\n    )\n\n    generated_tokens = model.generate(\n        **encoded,\n        forced_bos_token_id=tokenizer.get_lang_id(\"ta\")\n    )\n\n    preds = tokenizer.batch_decode(\n        generated_tokens,\n        skip_special_tokens=True\n    )\n\n    m2m_predictions.extend(preds)\n\nprint(\"Total predictions:\", len(m2m_predictions))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:24:39.242373Z","iopub.execute_input":"2026-04-13T14:24:39.243033Z","iopub.status.idle":"2026-04-13T14:38:05.293922Z","shell.execute_reply.started":"2026-04-13T14:24:39.243004Z","shell.execute_reply":"2026-04-13T14:38:05.292653Z"}},"outputs":[{"name":"stdout","text":"Total predictions: 276\n","output_type":"stream"}],"execution_count":32},{"cell_type":"code","source":"m2m_predictions = [preprocess(t) for t in m2m_predictions]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:50:58.063734Z","iopub.execute_input":"2026-04-13T14:50:58.064586Z","iopub.status.idle":"2026-04-13T14:50:58.070956Z","shell.execute_reply.started":"2026-04-13T14:50:58.064557Z","shell.execute_reply":"2026-04-13T14:50:58.070064Z"}},"outputs":[],"execution_count":33},{"cell_type":"code","source":"import evaluate\nimport sacrebleu\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.metrics.pairwise import cosine_similarity\nimport numpy as np\nfrom bert_score import score\n\nprint(\"Evaluating translations...\\n\")\n\n# CHANGE THIS depending on model\npredictions = m2m_predictions\n\n# Remove empty sentences\nclean_refs = []\nclean_preds = []\n\nfor ref, pred in zip(references, predictions):\n    if ref.strip() != \"\" and pred.strip() != \"\":\n        clean_refs.append(ref)\n        clean_preds.append(pred)\n\nprint(\"Valid samples:\", len(clean_preds))\n\n# BLEU\nbleu = evaluate.load(\"bleu\")\n\nreferences_bleu = [[ref] for ref in clean_refs]\n\nbleu_score = bleu.compute(\n    predictions=clean_preds,\n    references=references_bleu\n)\n\n# chrF\nchrf = sacrebleu.corpus_chrf(\n    clean_preds,\n    [clean_refs]\n)\n\n# BERTScore\nP, R, F1 = score(\n    clean_preds,\n    clean_refs,\n    model_type=\"bert-base-multilingual-cased\"\n)\n\nbert_f1 = F1.mean().item()\n\n# Embedding Similarity\nembed_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n\nref_embeddings = embed_model.encode(clean_refs)\npred_embeddings = embed_model.encode(clean_preds)\n\nsimilarities = cosine_similarity(\n    ref_embeddings,\n    pred_embeddings\n).diagonal()\n\nembedding_similarity = np.mean(similarities)\n\n# Print Results\nprint(\"\\n===== Evaluation Results =====\\n\")\n\nprint(\"BLEU Score:\", bleu_score[\"bleu\"])\nprint(\"chrF Score:\", chrf.score)\nprint(\"BERTScore F1:\", bert_f1)\nprint(\"Embedding Similarity:\", embedding_similarity)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:51:00.596479Z","iopub.execute_input":"2026-04-13T14:51:00.597264Z","iopub.status.idle":"2026-04-13T14:51:04.850673Z","shell.execute_reply.started":"2026-04-13T14:51:00.597236Z","shell.execute_reply":"2026-04-13T14:51:04.849305Z"}},"outputs":[{"name":"stdout","text":"Evaluating translations...\n\nValid samples: 276\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/199 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4328a08f769745fdaea8c59818a2b7cb"}},"metadata":{}},{"name":"stderr","text":"\u001b[1mBertModel LOAD REPORT\u001b[0m from: bert-base-multilingual-cased\nKey                                        | Status     |  | \n-------------------------------------------+------------+--+-\ncls.predictions.transform.dense.weight     | UNEXPECTED |  | \ncls.predictions.transform.LayerNorm.bias   | UNEXPECTED |  | \ncls.predictions.transform.LayerNorm.weight | UNEXPECTED |  | \ncls.seq_relationship.weight                | UNEXPECTED |  | \ncls.predictions.bias                       | UNEXPECTED |  | \ncls.seq_relationship.bias                  | UNEXPECTED |  | \ncls.predictions.transform.dense.bias       | UNEXPECTED |  | \n\n\u001b[3mNotes:\n- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/103 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a316a81f0dbb4124afd1ae9c72870c2c"}},"metadata":{}},{"name":"stderr","text":"\u001b[1mBertModel LOAD REPORT\u001b[0m from: sentence-transformers/all-MiniLM-L6-v2\nKey                     | Status     |  | \n------------------------+------------+--+-\nembeddings.position_ids | UNEXPECTED |  | \n\n\u001b[3mNotes:\n- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n","output_type":"stream"},{"name":"stdout","text":"\n===== Evaluation Results =====\n\nBLEU Score: 0.008726440899402313\nchrF Score: 17.461341772346902\nBERTScore F1: 0.6680243611335754\nEmbedding Similarity: 0.5583976\n","output_type":"stream"}],"execution_count":34}]}

notebooks/evaluate_models.py

@@ -0,0 +1,202 @@
+"""
+evaluation/evaluate_models.py
+─────────────────────────────────────────────────────────────────────────────
+Evaluates three pretrained MT models on the ai4bharat/IndicMTEval Tamil subset.
+
+Models evaluated:
+  1. facebook/nllb-200-distilled-600M  (NLLB)
+  2. facebook/m2m100_418M              (M2M100)
+  3. t5-base                           (T5)
+
+Metrics:
+  - BLEU
+  - chrF
+  - BERTScore F1
+  - Sentence Embedding Cosine Similarity
+
+Usage:
+  python evaluate_models.py [--model nllb|m2m|t5|all] [--samples 200]
+"""
+
+import re
+import argparse
+import numpy as np
+import evaluate
+import sacrebleu
+from datasets import load_dataset
+from transformers import (
+    NllbTokenizer, AutoModelForSeq2SeqLM,
+    M2M100Tokenizer, M2M100ForConditionalGeneration,
+    T5Tokenizer, T5ForConditionalGeneration,
+)
+from bert_score import score as bert_score
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+import torch
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+# ─────────────────────────────────────────
+# Preprocessing
+# ─────────────────────────────────────────
+def preprocess(text: str) -> str:
+    text = text.lower()
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()
+
+
+# ─────────────────────────────────────────
+# Dataset
+# ─────────────────────────────────────────
+def load_tamil_data(num_samples: int = 200):
+    print("Loading IndicMTEval dataset...")
+    dataset = load_dataset("ai4bharat/IndicMTEval", split="test")
+    dataset_tamil = dataset.filter(lambda x: x["language"] == "Tamil")
+    if num_samples:
+        dataset_tamil = dataset_tamil.select(range(min(num_samples, len(dataset_tamil))))
+    sources = [preprocess(t) for t in dataset_tamil["src"]]
+    references = [preprocess(t) for t in dataset_tamil["ref"]]
+    print(f"Loaded {len(sources)} Tamil samples.")
+    return sources, references
+
+
+# ─────────────────────────────────────────
+# Model Translators(nllb)
+# ─────────────────────────────────────────
+def translate_nllb(sources, batch_size=8):
+    print("\n[NLLB] Loading model...")
+    tokenizer = NllbTokenizer.from_pretrained("facebook/nllb-200-distilled-600M")
+    model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-600M").to(DEVICE)
+    model.eval()
+    preds = []
+    for i in range(0, len(sources), batch_size):
+        batch = sources[i:i+batch_size]
+        inputs = tokenizer(batch, return_tensors="pt", padding=True, truncation=True).to(DEVICE)
+        with torch.no_grad():
+            out = model.generate(
+                **inputs,
+                forced_bos_token_id=tokenizer.convert_tokens_to_ids("tam_Taml"),
+                num_beams=4, max_length=256,
+            )
+        preds.extend(tokenizer.batch_decode(out, skip_special_tokens=True))
+        print(f"  NLLB: {min(i+batch_size, len(sources))}/{len(sources)}", end="\r")
+    return [preprocess(p) for p in preds]
+
+
+def translate_m2m(sources, batch_size=8):
+    print("\n[M2M100] Loading model...")
+    tokenizer = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M")
+    model = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M").to(DEVICE)
+    model.eval()
+    tokenizer.src_lang = "en"
+    preds = []
+    for i in range(0, len(sources), batch_size):
+        batch = sources[i:i+batch_size]
+        encoded = tokenizer(batch, return_tensors="pt", padding=True, truncation=True).to(DEVICE)
+        with torch.no_grad():
+            out = model.generate(
+                **encoded,
+                forced_bos_token_id=tokenizer.get_lang_id("ta"),
+                num_beams=4, max_length=256,
+            )
+        preds.extend(tokenizer.batch_decode(out, skip_special_tokens=True))
+        print(f"  M2M: {min(i+batch_size, len(sources))}/{len(sources)}", end="\r")
+    return [preprocess(p) for p in preds]
+
+
+def translate_t5(sources, batch_size=8):
+    print("\n[T5] Loading model...")
+    tokenizer = T5Tokenizer.from_pretrained("t5-base")
+    model = T5ForConditionalGeneration.from_pretrained("t5-base").to(DEVICE)
+    model.eval()
+    preds = []
+    for i in range(0, len(sources), batch_size):
+        batch = sources[i:i+batch_size]
+        inputs = tokenizer(
+            ["translate English to Tamil: " + s for s in batch],
+            return_tensors="pt", padding=True, truncation=True,
+        ).to(DEVICE)
+        with torch.no_grad():
+            out = model.generate(**inputs, max_length=256)
+        preds.extend(tokenizer.batch_decode(out, skip_special_tokens=True))
+        print(f"  T5: {min(i+batch_size, len(sources))}/{len(sources)}", end="\r")
+    return [preprocess(p) for p in preds]
+
+
+# ─────────────────────────────────────────
+# Evaluation
+# ─────────────────────────────────────────
+def evaluate_predictions(predictions, references, model_name="Model"):
+    # Filter empty pairs
+    pairs = [(p, r) for p, r in zip(predictions, references) if p.strip() and r.strip()]
+    clean_preds, clean_refs = zip(*pairs)
+
+    print(f"\n  Evaluating {model_name} on {len(clean_preds)} valid samples...")
+
+    # BLEU
+    bleu = evaluate.load("bleu")
+    bleu_score = bleu.compute(predictions=list(clean_preds), references=[[r] for r in clean_refs])["bleu"]
+
+    # chrF
+    chrf_score = sacrebleu.corpus_chrf(list(clean_preds), [list(clean_refs)]).score
+
+    # BERTScore
+    _, _, F1 = bert_score(list(clean_preds), list(clean_refs), model_type="bert-base-multilingual-cased")
+    bert_f1 = F1.mean().item()
+
+    # Embedding Cosine Similarity
+    embed_model = SentenceTransformer("all-MiniLM-L6-v2")
+    ref_emb = embed_model.encode(list(clean_refs))
+    pred_emb = embed_model.encode(list(clean_preds))
+    cos_sim = cosine_similarity(ref_emb, pred_emb).diagonal().mean()
+
+    return {
+        "model": model_name,
+        "BLEU": round(bleu_score, 4),
+        "chrF": round(chrf_score, 2),
+        "BERTScore_F1": round(bert_f1, 4),
+        "EmbeddingSim": round(float(cos_sim), 4),
+    }
+
+
+# ─────────────────────────────────────────
+# Main
+# ─────────────────────────────────────────
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", default="all", choices=["nllb", "m2m", "t5", "all"])
+    parser.add_argument("--samples", type=int, default=200)
+    args = parser.parse_args()
+
+    sources, references = load_tamil_data(args.samples)
+
+    results = []
+
+    if args.model in ("nllb", "all"):
+        preds = translate_nllb(sources)
+        results.append(evaluate_predictions(preds, references, "NLLB-200 (600M)"))
+
+    if args.model in ("m2m", "all"):
+        preds = translate_m2m(sources)
+        results.append(evaluate_predictions(preds, references, "M2M100 (418M)"))
+
+    if args.model in ("t5", "all"):
+        preds = translate_t5(sources)
+        results.append(evaluate_predictions(preds, references, "T5-Base"))
+
+    # Summary table
+    print("\n" + "="*65)
+    print(f"{'Model':<22} {'BLEU':>8} {'chrF':>8} {'BERTScore':>10} {'CosSim':>8}")
+    print("-"*65)
+    for r in results:
+        print(f"{r['model']:<22} {r['BLEU']:>8.4f} {r['chrF']:>8.2f} {r['BERTScore_F1']:>10.4f} {r['EmbeddingSim']:>8.4f}")
+    print("="*65)
+
+    # Best model
+    best = max(results, key=lambda x: x["chrF"])
+    print(f"\n🏆 Best model: {best['model']} (chrF: {best['chrF']})")
+
+
+if __name__ == "__main__":
+    main()

notebooks/evaluate_multilingual.py

@@ -0,0 +1,218 @@
+"""
+evaluation/evaluate_multilingual.py
+─────────────────────────────────────────────────────────────────────────────
+Evaluates NLLB-200 on English → 5 Indian languages using IndicMTEval.
+
+Languages : Tamil · Hindi · Telugu · Kannada · Malayalam
+Model      : facebook/nllb-200-distilled-600M
+Metrics    : BLEU · chrF · BERTScore F1 · Cosine Similarity
+
+Design rationale
+────────────────
+Primary evaluation (model selection, paper results) remains English → Tamil
+because IndicMTEval has the richest human quality scores (MQM / DA) for Tamil.
+This script adds NLLB capability demonstration across all five languages so
+the final report can show multilingual breadth alongside the Tamil deep-dive.
+
+Usage
+─────
+  # Evaluate all languages (≈200 samples each, default)
+  python evaluate_multilingual.py
+
+  # Single language fast check
+  python evaluate_multilingual.py --lang Tamil --samples 50
+
+  # Save results to JSON
+  python evaluate_multilingual.py --output results/multilingual_eval.json
+"""
+
+import re
+import json
+import argparse
+import numpy as np
+import evaluate
+import sacrebleu
+from datasets import load_dataset
+from transformers import NllbTokenizer, AutoModelForSeq2SeqLM
+from bert_score import score as bert_score_fn
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+import torch
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Language → NLLB token + IndicMTEval filter string + BERTScore lang code
+# ─────────────────────────────────────────────────────────────────────────────
+LANG_CONFIG = {
+    "Tamil":     {"token": "tam_Taml", "filter": "Tamil",     "bert_lang": "ta"},
+    "Hindi":     {"token": "hin_Deva", "filter": "Hindi",     "bert_lang": "hi"},
+    "Telugu":    {"token": "tel_Telu", "filter": "Telugu",    "bert_lang": "te"},
+    "Kannada":   {"token": "kan_Knda", "filter": "Kannada",   "bert_lang": "kn"},
+    "Malayalam": {"token": "mal_Mlym", "filter": "Malayalam", "bert_lang": "ml"},
+}
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Helpers
+# ─────────────────────────────────────────────────────────────────────────────
+def preprocess(text: str) -> str:
+    return re.sub(r"\s+", " ", text.lower()).strip()
+
+
+def load_lang_data(language: str, num_samples: int):
+    """Load and filter IndicMTEval for a given language."""
+    cfg = LANG_CONFIG[language]
+    dataset = load_dataset("ai4bharat/IndicMTEval", split="test")
+    subset = dataset.filter(lambda x: x["language"] == cfg["filter"])
+    n = min(num_samples, len(subset))
+    subset = subset.select(range(n))
+    sources    = [preprocess(t) for t in subset["src"]]
+    references = [preprocess(t) for t in subset["ref"]]
+    print(f"  [{language}] {n} samples loaded.")
+    return sources, references
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Translation  (model + tokenizer passed in so they load only once)
+# ─────────────────────────────────────────────────────────────────────────────
+def translate_batch(sources, nllb_token, tokenizer, model, batch_size=8):
+    predictions = []
+    total = len(sources)
+    for i in range(0, total, batch_size):
+        batch = sources[i:i + batch_size]
+        inputs = tokenizer(
+            batch,
+            return_tensors="pt",
+            padding=True,
+            truncation=True,
+            max_length=512,
+        ).to(DEVICE)
+        with torch.no_grad():
+            out = model.generate(
+                **inputs,
+                forced_bos_token_id=tokenizer.convert_tokens_to_ids(nllb_token),
+                num_beams=4,
+                max_length=256,
+                early_stopping=True,
+            )
+        predictions.extend(tokenizer.batch_decode(out, skip_special_tokens=True))
+        done = min(i + batch_size, total)
+        print(f"    Translated {done}/{total}", end="\r")
+    print()
+    return [preprocess(p) for p in predictions]
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Evaluation
+# ─────────────────────────────────────────────────────────────────────────────
+def compute_metrics(predictions, references, bert_lang: str):
+    # Drop empty pairs
+    pairs = [(p, r) for p, r in zip(predictions, references) if p.strip() and r.strip()]
+    if not pairs:
+        return {"bleu": 0, "chrf": 0, "bert_f1": 0, "cosine": 0, "n_valid": 0}
+
+    preds, refs = zip(*pairs)
+    preds, refs = list(preds), list(refs)
+
+    # BLEU
+    bleu_metric = evaluate.load("bleu")
+    bleu_val = bleu_metric.compute(
+        predictions=preds,
+        references=[[r] for r in refs],
+    )["bleu"]
+
+    # chrF
+    chrf_val = sacrebleu.corpus_chrf(preds, [refs]).score
+
+    # BERTScore
+    _, _, F1 = bert_score_fn(preds, refs, model_type="bert-base-multilingual-cased")
+    bert_f1 = F1.mean().item()
+
+    # Cosine similarity
+    embed_model = SentenceTransformer("all-MiniLM-L6-v2")
+    ref_emb  = embed_model.encode(refs)
+    pred_emb = embed_model.encode(preds)
+    cos_sim  = cosine_similarity(ref_emb, pred_emb).diagonal().mean()
+
+    return {
+        "bleu":    round(bleu_val, 4),
+        "chrf":    round(chrf_val, 2),
+        "bert_f1": round(bert_f1, 4),
+        "cosine":  round(float(cos_sim), 4),
+        "n_valid": len(preds),
+    }
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Main
+# ─────────────────────────────────────────────────────────────────────────────
+def main():
+    parser = argparse.ArgumentParser(description="Multilingual MT Evaluation — NLLB-200")
+    parser.add_argument("--lang",    default="all",
+                        choices=list(LANG_CONFIG.keys()) + ["all"],
+                        help="Language to evaluate (default: all)")
+    parser.add_argument("--samples", type=int, default=200,
+                        help="Max samples per language (default: 200)")
+    parser.add_argument("--output",  default=None,
+                        help="Optional JSON file to save results")
+    args = parser.parse_args()
+
+    languages = list(LANG_CONFIG.keys()) if args.lang == "all" else [args.lang]
+
+    # Load model ONCE — reused across all languages
+    print(f"\nLoading NLLB model on {DEVICE}...")
+    tokenizer = NllbTokenizer.from_pretrained("facebook/nllb-200-distilled-600M")
+    model     = AutoModelForSeq2SeqLM.from_pretrained(
+        "facebook/nllb-200-distilled-600M"
+    ).to(DEVICE)
+    model.eval()
+    print("Model ready ✓\n")
+
+    all_results = {}
+
+    for lang in languages:
+        cfg = LANG_CONFIG[lang]
+        print(f"{'─'*60}")
+        print(f"  Language : {lang}  ({cfg['token']})")
+        print(f"{'─'*60}")
+
+        sources, references = load_lang_data(lang, args.samples)
+
+        print(f"  Translating {len(sources)} sentences…")
+        preds = translate_batch(sources, cfg["token"], tokenizer, model)
+
+        print(f"  Computing metrics…")
+        metrics = compute_metrics(preds, references, cfg["bert_lang"])
+        metrics["language"] = lang
+        metrics["nllb_token"] = cfg["token"]
+
+        all_results[lang] = metrics
+        print(f"  BLEU={metrics['bleu']:.4f}  chrF={metrics['chrf']:.2f}  "
+              f"BERT={metrics['bert_f1']:.4f}  Cosine={metrics['cosine']:.4f}  "
+              f"(n={metrics['n_valid']})\n")
+
+    # ── Summary table ─────────────────────────────────────────────────────────
+    print("\n" + "=" * 72)
+    print(f"{'Language':<14} {'NLLB Token':<14} {'BLEU':>8} {'chrF':>8} {'BERTScore':>10} {'CosSim':>8} {'N':>6}")
+    print("-" * 72)
+    for lang, r in all_results.items():
+        print(f"{lang:<14} {r['nllb_token']:<14} {r['bleu']:>8.4f} "
+              f"{r['chrf']:>8.2f} {r['bert_f1']:>10.4f} {r['cosine']:>8.4f} {r['n_valid']:>6}")
+    print("=" * 72)
+
+    best_lang = max(all_results, key=lambda k: all_results[k]["chrf"])
+    print(f"\n🏆 Best language result: {best_lang} (chrF: {all_results[best_lang]['chrf']})")
+    print("   (Tamil is the primary evaluation language for model comparison.)\n")
+
+    # ── Save JSON ─────────────────────────────────────────────────────────────
+    if args.output:
+        import os
+        os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
+        with open(args.output, "w", encoding="utf-8") as f:
+            json.dump(all_results, f, indent=2, ensure_ascii=False)
+        print(f"Results saved → {args.output}")
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,11 @@
+gradio>=4.0.0
+transformers>=4.35.0
+torch>=2.0.0
+sentencepiece>=0.1.99
+sacrebleu>=2.3.1
+evaluate>=0.4.0
+bert-score>=0.3.13
+sentence-transformers>=2.2.2
+scikit-learn>=1.3.0
+numpy>=1.24.0
+datasets>=2.14.0