Upload 31 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+models/checkpoint-30/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+models/tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,71 @@
----
-title: Project
-emoji: ⚡
-colorFrom: purple
-colorTo: red
-sdk: gradio
-sdk_version: 6.3.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Multilingual Transliteration
+emoji: 🌐
+colorFrom: blue
+colorTo: purple
+sdk: gradio
+sdk_version: 5.8.0
+app_file: app.py
+pinned: false
+---
+
+# Multilingual Transliteration Model
+
+This project implements a multilingual transliteration model (English -> Hindi, Bengali, Tamil) using a fine-tuned mT5 model. It focuses on optimization using CTranslate2 for fast inference and provides a Gradio-based web interface.
+
+## Project Structure
+- `src/`: Source code for training, optimization, and deployment.
+- `data/`: Directory for storing datasets (train/test/val).
+- `models/`: Directory for saving trained and optimized models.
+- `requirements.txt`: Python dependencies.
+
+## Setup
+
+1.  **Clone the repository:**
+    ```bash
+    git clone <repo_url>
+    cd <repo_name>
+    ```
+
+2.  **Create a virtual environment (optional but recommended):**
+    ```bash
+    python -m venv venv
+    .\venv\Scripts\activate  # Windows
+    # source venv/bin/activate # Linux/Mac
+    ```
+
+3.  **Install dependencies:**
+    ```bash
+    pip install -r requirements.txt
+    ```
+
+## Usage
+
+### 1. Data Preparation
+Generate dummy data for training:
+```bash
+python src/prepare_data.py
+```
+
+### 2. Training
+Train the mT5 model:
+```bash
+python src/train.py
+```
+
+### 3. Optimization
+Optimize the trained model using CTranslate2 and benchmark:
+```bash
+python src/optimize.py
+```
+
+### 4. Run Demo
+Launch the Gradio app:
+```bash
+python src/app.py
+```
+
+## Approach
+- **Model:** `google/mt5-small` is used as the base model due to its multilingual capabilities and efficiency.
+- **Optimization:** CTranslate2 is used to quantize and optimize the model for faster CPU/GPU inference.
+- **Deployment:** Gradio provides a simple and interactive UI for the model.

app.py

@@ -0,0 +1,118 @@
+import gradio as gr
+import ctranslate2
+import transformers
+import os
+
+MODEL_DIR = "models"
+TOKENIZER_DIR = "models" # Relative path for HF Space compatibility
+
+# Check if optimized model exists, else fallback or warn
+if not os.path.exists(MODEL_DIR):
+    print("Warning: CT2 Model not found. Please run src/optimize.py")
+
+# Load Global resources
+def load_model():
+    global translator, tokenizer
+    try:
+        # 1. Try to load CTranslate2 model (Optimized Local)
+        if os.path.exists(os.path.join(MODEL_DIR, "model.bin")):
+            print("Loading CTranslate2 model from local storage...")
+            translator = ctranslate2.Translator(MODEL_DIR)
+            tokenizer = transformers.MBart50TokenizerFast.from_pretrained(TOKENIZER_DIR)
+        
+        # 2. Fallback: Load from Hugging Face Hub
+        else:
+            print("Local weights not found. Downloading fallback model from HF Hub (facebook/mbart-large-50)...")
+            from transformers import MBartForConditionalGeneration, MBart50TokenizerFast
+            base_model_id = "facebook/mbart-large-50-many-to-many-mmt"
+            tokenizer = MBart50TokenizerFast.from_pretrained(base_model_id)
+            hf_model = MBartForConditionalGeneration.from_pretrained(base_model_id)
+            
+            # Create a simple wrapper to make hf_model act like a CT2 translator for the existing code
+            class TransformersWrapper:
+                def __init__(self, model, tokenizer):
+                    self.model = model
+                    self.tokenizer = tokenizer
+                def translate_batch(self, source_tokens, target_prefix):
+                    # Convert tokens back to text for transformers
+                    text = [self.tokenizer.decode(self.tokenizer.convert_tokens_to_ids(s)) for s in source_tokens]
+                    encoded = self.tokenizer(text, return_tensors="pt", padding=True)
+                    # Get target lang code
+                    forced_bos_token_id = self.tokenizer.lang_code_to_id[target_prefix[0][0]]
+                    generated_tokens = self.model.generate(
+                        **encoded, 
+                        forced_bos_token_id=forced_bos_token_id
+                    )
+                    # Wrap in a result object that mimics CT2 output
+                    class Result:
+                        def __init__(self, tokens): self.hypotheses = [tokens]
+                    
+                    return [Result(self.tokenizer.convert_ids_to_tokens(g)) for g in generated_tokens]
+            
+            translator = TransformersWrapper(hf_model, tokenizer)
+            print("Fallback model loaded successfully.")
+
+    except Exception as e:
+        print(f"Error loading model: {e}")
+        translator = None
+        tokenizer = None
+
+load_model()
+if tokenizer:
+    tokenizer.src_lang = "en_XX"
+
+LANG_CODES = {
+    "Hindi": "hi_IN",
+    "Bengali": "bn_IN",
+    "Tamil": "ta_IN"
+}
+
+def transliterate(text, target_language):
+    if not translator or not text:
+        return "Model not loaded or empty input."
+    
+    target_code = LANG_CODES.get(target_language)
+    if not target_code:
+        return "Invalid Language"
+
+    # Tokenize
+    source = tokenizer.convert_ids_to_tokens(tokenizer.encode(text))
+    
+    # Translate
+    results = translator.translate_batch(
+        [source],
+        target_prefix=[[target_code]]
+    )
+    
+    # Decode
+    target = results[0].hypotheses[0]
+    return tokenizer.decode(tokenizer.convert_tokens_to_ids(target), skip_special_tokens=True)
+
+def create_demo():
+    with gr.Blocks(theme=gr.themes.Soft()) as demo:
+        gr.Markdown("# 🌐 Multilingual Transliteration Model")
+        
+        gr.Markdown("Transliterate English text to Hindi, Bengali, or Tamil.")
+        
+        with gr.Row():
+            with gr.Column():
+                input_text = gr.Textbox(label="Input Text (English/Roman)", placeholder="e.g. Namaste", lines=3)
+                target_lang = gr.Dropdown(choices=["Hindi", "Bengali", "Tamil"], value="Hindi", label="Target Language")
+                btn = gr.Button("🚀 Transliterate", variant="primary")
+            
+            with gr.Column():
+                output_text = gr.Textbox(label="Transliterated Output", lines=5)
+        
+        gr.Examples(
+            examples=[
+                ["Namaste", "Hindi"],
+                ["Kemon achen", "Bengali"],
+                ["Vanakkam", "Tamil"]
+            ],
+            inputs=[input_text, target_lang]
+        )
+        
+        btn.click(fn=transliterate, inputs=[input_text, target_lang], outputs=output_text)
+        
+    return demo
+

data/test.csv

@@ -0,0 +1,16 @@
+source,target,lang
+aap,आप,hi
+hai,है,hi
+namoshkar,নমস্কার,bn
+amar,আমার,bn
+vanakkam,வணக்கம்,ta
+jal,জল,bn
+nadu,நாடு,ta
+amar,আমার,bn
+namaste,नमस्ते,hi
+kar,कर,hi
+jal,জল,bn
+namoshkar,নমস্কার,bn
+nam,নাম,bn
+nam,নাম,bn
+kya,क्या,hi

data/train.csv

@@ -0,0 +1,121 @@
+source,target,lang
+irukkeenga,இருக்கிறீர்கள்,ta
+naam,नाम,hi
+thanni,தண்ணீர்,ta
+aap,आप,hi
+nam,নাম,bn
+naam,नाम,hi
+achen,আছেন,bn
+bharat,भारत,hi
+peyar,பெயர்,ta
+naam,नाम,hi
+bharat,भारत,hi
+kya,क्या,hi
+en,என்,ta
+nadu,நாடு,ta
+eppadi,எப்படி,ta
+amar,আমার,bn
+en,என்,ta
+kemon,কেমন,bn
+achen,আছেন,bn
+achen,আছেন,bn
+ho,हो,hi
+naam,नाम,hi
+ho,हो,hi
+namaste,नमस्ते,hi
+neengal,நீங்கள்,ta
+bangla,বাংলা,bn
+sapadu,சாப்பாடு,ta
+bharat,भारत,hi
+kya,क्या,hi
+achen,আছেন,bn
+thanni,தண்ணீர்,ta
+khabar,খাবার,bn
+kya,क्या,hi
+mera,मेरा,hi
+vanakkam,வணக்கம்,ta
+bangla,বাংলা,bn
+peyar,பெயர்,ta
+thanni,தண்ணீர்,ta
+hai,है,hi
+irukkeenga,இருக்கிறீர்கள்,ta
+neengal,நீங்கள்,ta
+bangla,বাংলা,bn
+vanakkam,வணக்கம்,ta
+namaste,नमस्ते,hi
+mera,मेरा,hi
+kar,कर,hi
+bangla,বাংলা,bn
+aap,आप,hi
+en,என்,ta
+eppadi,எப்படி,ta
+ho,हो,hi
+en,என்,ta
+desh,দেশ,bn
+amar,আমার,bn
+sapadu,சாப்பாடு,ta
+neengal,நீங்கள்,ta
+kya,क्या,hi
+tamil,தமிழ்,ta
+apni,আপনি,bn
+nam,নাম,bn
+bharat,भारत,hi
+tamil,தமிழ்,ta
+neengal,நீங்கள்,ta
+khabar,খাবার,bn
+rahe,रहे,hi
+eppadi,எப்படி,ta
+apni,আপনি,bn
+aap,आप,hi
+jal,জল,bn
+eppadi,எப்படி,ta
+eppadi,எப்படி,ta
+kar,कर,hi
+khabar,খাবার,bn
+nadu,நாடு,ta
+irukkeenga,இருக்கிறீர்கள்,ta
+thanni,தண்ணீர்,ta
+mera,मेरा,hi
+tamil,தமிழ்,ta
+bangla,বাংলা,bn
+peyar,பெயர்,ta
+kemon,কেমন,bn
+tamil,தமிழ்,ta
+sapadu,சாப்பாடு,ta
+kemon,কেমন,bn
+irukkeenga,இருக்கிறீர்கள்,ta
+peyar,பெயர்,ta
+ho,हो,hi
+kar,कर,hi
+bharat,भारत,hi
+desh,দেশ,bn
+khabar,খাবার,bn
+khabar,খাবার,bn
+apni,আপনি,bn
+desh,দেশ,bn
+desh,দেশ,bn
+namoshkar,নমস্কার,bn
+namaste,नमस्ते,hi
+kemon,কেমন,bn
+rahe,रहे,hi
+jal,জল,bn
+rahe,रहे,hi
+rahe,रहे,hi
+thanni,தண்ணீர்,ta
+mera,मेरा,hi
+mera,मेरा,hi
+en,என்,ta
+sapadu,சாப்பாடு,ta
+kemon,কেমন,bn
+kar,कर,hi
+tamil,தமிழ்,ta
+vanakkam,வணக்கம்,ta
+naam,नाम,hi
+desh,দেশ,bn
+namaste,नमस्ते,hi
+nadu,நாடு,ta
+jal,জল,bn
+nadu,நாடு,ta
+aap,आप,hi
+hai,है,hi
+namoshkar,নমস্কার,bn

data/val.csv

@@ -0,0 +1,16 @@
+source,target,lang
+amar,আমার,bn
+apni,আপনি,bn
+sapadu,சாப்பாடு,ta
+neengal,நீங்கள்,ta
+irukkeenga,இருக்கிறீர்கள்,ta
+peyar,பெயர்,ta
+rahe,रहे,hi
+hai,है,hi
+namoshkar,নমস্কার,bn
+nam,নাম,bn
+achen,আছেন,bn
+ho,हो,hi
+hai,है,hi
+apni,আপনি,bn
+vanakkam,வணக்கம்,ta

models/checkpoint-30/config.json

@@ -0,0 +1,57 @@
+{
+  "_num_labels": 3,
+  "activation_dropout": 0.0,
+  "activation_function": "relu",
+  "add_bias_logits": false,
+  "add_final_layer_norm": true,
+  "architectures": [
+    "MBartForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "classif_dropout": 0.0,
+  "classifier_dropout": 0.0,
+  "d_model": 1024,
+  "decoder_attention_heads": 16,
+  "decoder_ffn_dim": 4096,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 12,
+  "decoder_start_token_id": 2,
+  "dropout": 0.1,
+  "dtype": "float32",
+  "early_stopping": null,
+  "encoder_attention_heads": 16,
+  "encoder_ffn_dim": 4096,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 12,
+  "eos_token_id": 2,
+  "forced_eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "max_length": null,
+  "max_position_embeddings": 1024,
+  "model_type": "mbart",
+  "normalize_before": true,
+  "normalize_embedding": true,
+  "num_beams": null,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "scale_embedding": true,
+  "static_position_embeddings": false,
+  "tokenizer_class": "MBart50Tokenizer",
+  "transformers_version": "4.57.3",
+  "use_cache": true,
+  "vocab_size": 250054
+}

models/checkpoint-30/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "decoder_start_token_id": 2,
+  "early_stopping": true,
+  "eos_token_id": [
+    2
+  ],
+  "forced_eos_token_id": 2,
+  "max_length": 200,
+  "num_beams": 5,
+  "pad_token_id": 1,
+  "transformers_version": "4.57.3"
+}

models/checkpoint-30/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf06980fc3200df90cdd62120cbad96ec7378e2bb8faae0509e98d67fea85727
+size 14645

models/checkpoint-30/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:973e7699cf118c0ef2f285910efd67abb42d0d1ae7bae40cb22396d19a64328c
+size 1383

models/checkpoint-30/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c01134b5ae1edcac974086698aba68af7d61c087c24b035fd0502482c1fac02
+size 1465

models/checkpoint-30/sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

models/checkpoint-30/special_tokens_map.json

@@ -0,0 +1,69 @@
+{
+  "additional_special_tokens": [
+    "ar_AR",
+    "cs_CZ",
+    "de_DE",
+    "en_XX",
+    "es_XX",
+    "et_EE",
+    "fi_FI",
+    "fr_XX",
+    "gu_IN",
+    "hi_IN",
+    "it_IT",
+    "ja_XX",
+    "kk_KZ",
+    "ko_KR",
+    "lt_LT",
+    "lv_LV",
+    "my_MM",
+    "ne_NP",
+    "nl_XX",
+    "ro_RO",
+    "ru_RU",
+    "si_LK",
+    "tr_TR",
+    "vi_VN",
+    "zh_CN",
+    "af_ZA",
+    "az_AZ",
+    "bn_IN",
+    "fa_IR",
+    "he_IL",
+    "hr_HR",
+    "id_ID",
+    "ka_GE",
+    "km_KH",
+    "mk_MK",
+    "ml_IN",
+    "mn_MN",
+    "mr_IN",
+    "pl_PL",
+    "ps_AF",
+    "pt_XX",
+    "sv_SE",
+    "sw_KE",
+    "ta_IN",
+    "te_IN",
+    "th_TH",
+    "tl_XX",
+    "uk_UA",
+    "ur_PK",
+    "xh_ZA",
+    "gl_ES",
+    "sl_SI"
+  ],
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

models/checkpoint-30/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0da4e7af9b86e84c844ce9b0d58a845dd3b0d9724abef93bc226aeb17d5110a0
+size 17110186

models/checkpoint-30/tokenizer_config.json

@@ -0,0 +1,529 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "ar_AR",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250002": {
+      "content": "cs_CZ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250003": {
+      "content": "de_DE",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250004": {
+      "content": "en_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250005": {
+      "content": "es_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250006": {
+      "content": "et_EE",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250007": {
+      "content": "fi_FI",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250008": {
+      "content": "fr_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250009": {
+      "content": "gu_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250010": {
+      "content": "hi_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250011": {
+      "content": "it_IT",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250012": {
+      "content": "ja_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250013": {
+      "content": "kk_KZ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250014": {
+      "content": "ko_KR",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250015": {
+      "content": "lt_LT",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250016": {
+      "content": "lv_LV",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250017": {
+      "content": "my_MM",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250018": {
+      "content": "ne_NP",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250019": {
+      "content": "nl_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250020": {
+      "content": "ro_RO",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250021": {
+      "content": "ru_RU",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250022": {
+      "content": "si_LK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250023": {
+      "content": "tr_TR",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250024": {
+      "content": "vi_VN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250025": {
+      "content": "zh_CN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250026": {
+      "content": "af_ZA",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250027": {
+      "content": "az_AZ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250028": {
+      "content": "bn_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250029": {
+      "content": "fa_IR",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250030": {
+      "content": "he_IL",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250031": {
+      "content": "hr_HR",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250032": {
+      "content": "id_ID",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250033": {
+      "content": "ka_GE",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250034": {
+      "content": "km_KH",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250035": {
+      "content": "mk_MK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250036": {
+      "content": "ml_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250037": {
+      "content": "mn_MN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250038": {
+      "content": "mr_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250039": {
+      "content": "pl_PL",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250040": {
+      "content": "ps_AF",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250041": {
+      "content": "pt_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250042": {
+      "content": "sv_SE",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250043": {
+      "content": "sw_KE",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250044": {
+      "content": "ta_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250045": {
+      "content": "te_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250046": {
+      "content": "th_TH",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250047": {
+      "content": "tl_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250048": {
+      "content": "uk_UA",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250049": {
+      "content": "ur_PK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250050": {
+      "content": "xh_ZA",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250051": {
+      "content": "gl_ES",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250052": {
+      "content": "sl_SI",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250053": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "ar_AR",
+    "cs_CZ",
+    "de_DE",
+    "en_XX",
+    "es_XX",
+    "et_EE",
+    "fi_FI",
+    "fr_XX",
+    "gu_IN",
+    "hi_IN",
+    "it_IT",
+    "ja_XX",
+    "kk_KZ",
+    "ko_KR",
+    "lt_LT",
+    "lv_LV",
+    "my_MM",
+    "ne_NP",
+    "nl_XX",
+    "ro_RO",
+    "ru_RU",
+    "si_LK",
+    "tr_TR",
+    "vi_VN",
+    "zh_CN",
+    "af_ZA",
+    "az_AZ",
+    "bn_IN",
+    "fa_IR",
+    "he_IL",
+    "hr_HR",
+    "id_ID",
+    "ka_GE",
+    "km_KH",
+    "mk_MK",
+    "ml_IN",
+    "mn_MN",
+    "mr_IN",
+    "pl_PL",
+    "ps_AF",
+    "pt_XX",
+    "sv_SE",
+    "sw_KE",
+    "ta_IN",
+    "te_IN",
+    "th_TH",
+    "tl_XX",
+    "uk_UA",
+    "ur_PK",
+    "xh_ZA",
+    "gl_ES",
+    "sl_SI"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "language_codes": "ML50",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "src_lang": "en_XX",
+  "tgt_lang": "hi_IN",
+  "tokenizer_class": "MBart50Tokenizer",
+  "unk_token": "<unk>"
+}

models/checkpoint-30/trainer_state.json

@@ -0,0 +1,63 @@
+{
+  "best_global_step": 30,
+  "best_metric": 7.648374557495117,
+  "best_model_checkpoint": "/content/drive/MyDrive/Nagina-2/models/mbart-transliteration/checkpoint-30",
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 30,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.3333333333333333,
+      "grad_norm": 94.37104797363281,
+      "learning_rate": 3.5e-05,
+      "loss": 11.4112,
+      "step": 10
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 101.393798828125,
+      "learning_rate": 1.8333333333333333e-05,
+      "loss": 9.2075,
+      "step": 20
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 103.45658111572266,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 8.0438,
+      "step": 30
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 7.648374557495117,
+      "eval_runtime": 0.2819,
+      "eval_samples_per_second": 53.202,
+      "eval_steps_per_second": 14.187,
+      "step": 30
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 30,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 32506946519040.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

models/checkpoint-30/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9013c117330a5e2e1042c93ca678d33d3f6c2afa498e8a5c8079ab49db2ccd69
+size 6033

models/config.json

@@ -0,0 +1,57 @@
+{
+  "_num_labels": 3,
+  "activation_dropout": 0.0,
+  "activation_function": "relu",
+  "add_bias_logits": false,
+  "add_final_layer_norm": true,
+  "architectures": [
+    "MBartForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "classif_dropout": 0.0,
+  "classifier_dropout": 0.0,
+  "d_model": 1024,
+  "decoder_attention_heads": 16,
+  "decoder_ffn_dim": 4096,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 12,
+  "decoder_start_token_id": 2,
+  "dropout": 0.1,
+  "dtype": "float32",
+  "early_stopping": null,
+  "encoder_attention_heads": 16,
+  "encoder_ffn_dim": 4096,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 12,
+  "eos_token_id": 2,
+  "forced_eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "max_length": null,
+  "max_position_embeddings": 1024,
+  "model_type": "mbart",
+  "normalize_before": true,
+  "normalize_embedding": true,
+  "num_beams": null,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "scale_embedding": true,
+  "static_position_embeddings": false,
+  "tokenizer_class": "MBart50Tokenizer",
+  "transformers_version": "4.57.3",
+  "use_cache": true,
+  "vocab_size": 250054
+}

models/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "decoder_start_token_id": 2,
+  "early_stopping": true,
+  "eos_token_id": [
+    2
+  ],
+  "forced_eos_token_id": 2,
+  "max_length": 200,
+  "num_beams": 5,
+  "pad_token_id": 1,
+  "transformers_version": "4.57.3"
+}

models/sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

models/special_tokens_map.json

@@ -0,0 +1,69 @@
+{
+  "additional_special_tokens": [
+    "ar_AR",
+    "cs_CZ",
+    "de_DE",
+    "en_XX",
+    "es_XX",
+    "et_EE",
+    "fi_FI",
+    "fr_XX",
+    "gu_IN",
+    "hi_IN",
+    "it_IT",
+    "ja_XX",
+    "kk_KZ",
+    "ko_KR",
+    "lt_LT",
+    "lv_LV",
+    "my_MM",
+    "ne_NP",
+    "nl_XX",
+    "ro_RO",
+    "ru_RU",
+    "si_LK",
+    "tr_TR",
+    "vi_VN",
+    "zh_CN",
+    "af_ZA",
+    "az_AZ",
+    "bn_IN",
+    "fa_IR",
+    "he_IL",
+    "hr_HR",
+    "id_ID",
+    "ka_GE",
+    "km_KH",
+    "mk_MK",
+    "ml_IN",
+    "mn_MN",
+    "mr_IN",
+    "pl_PL",
+    "ps_AF",
+    "pt_XX",
+    "sv_SE",
+    "sw_KE",
+    "ta_IN",
+    "te_IN",
+    "th_TH",
+    "tl_XX",
+    "uk_UA",
+    "ur_PK",
+    "xh_ZA",
+    "gl_ES",
+    "sl_SI"
+  ],
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

models/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0da4e7af9b86e84c844ce9b0d58a845dd3b0d9724abef93bc226aeb17d5110a0
+size 17110186

models/tokenizer_config.json

@@ -0,0 +1,529 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "ar_AR",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250002": {
+      "content": "cs_CZ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250003": {
+      "content": "de_DE",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250004": {
+      "content": "en_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250005": {
+      "content": "es_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250006": {
+      "content": "et_EE",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250007": {
+      "content": "fi_FI",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250008": {
+      "content": "fr_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250009": {
+      "content": "gu_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250010": {
+      "content": "hi_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250011": {
+      "content": "it_IT",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250012": {
+      "content": "ja_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250013": {
+      "content": "kk_KZ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250014": {
+      "content": "ko_KR",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250015": {
+      "content": "lt_LT",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250016": {
+      "content": "lv_LV",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250017": {
+      "content": "my_MM",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250018": {
+      "content": "ne_NP",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250019": {
+      "content": "nl_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250020": {
+      "content": "ro_RO",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250021": {
+      "content": "ru_RU",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250022": {
+      "content": "si_LK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250023": {
+      "content": "tr_TR",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250024": {
+      "content": "vi_VN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250025": {
+      "content": "zh_CN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250026": {
+      "content": "af_ZA",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250027": {
+      "content": "az_AZ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250028": {
+      "content": "bn_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250029": {
+      "content": "fa_IR",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250030": {
+      "content": "he_IL",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250031": {
+      "content": "hr_HR",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250032": {
+      "content": "id_ID",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250033": {
+      "content": "ka_GE",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250034": {
+      "content": "km_KH",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250035": {
+      "content": "mk_MK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250036": {
+      "content": "ml_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250037": {
+      "content": "mn_MN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250038": {
+      "content": "mr_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250039": {
+      "content": "pl_PL",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250040": {
+      "content": "ps_AF",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250041": {
+      "content": "pt_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250042": {
+      "content": "sv_SE",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250043": {
+      "content": "sw_KE",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250044": {
+      "content": "ta_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250045": {
+      "content": "te_IN",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250046": {
+      "content": "th_TH",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250047": {
+      "content": "tl_XX",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250048": {
+      "content": "uk_UA",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250049": {
+      "content": "ur_PK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250050": {
+      "content": "xh_ZA",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250051": {
+      "content": "gl_ES",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250052": {
+      "content": "sl_SI",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250053": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "ar_AR",
+    "cs_CZ",
+    "de_DE",
+    "en_XX",
+    "es_XX",
+    "et_EE",
+    "fi_FI",
+    "fr_XX",
+    "gu_IN",
+    "hi_IN",
+    "it_IT",
+    "ja_XX",
+    "kk_KZ",
+    "ko_KR",
+    "lt_LT",
+    "lv_LV",
+    "my_MM",
+    "ne_NP",
+    "nl_XX",
+    "ro_RO",
+    "ru_RU",
+    "si_LK",
+    "tr_TR",
+    "vi_VN",
+    "zh_CN",
+    "af_ZA",
+    "az_AZ",
+    "bn_IN",
+    "fa_IR",
+    "he_IL",
+    "hr_HR",
+    "id_ID",
+    "ka_GE",
+    "km_KH",
+    "mk_MK",
+    "ml_IN",
+    "mn_MN",
+    "mr_IN",
+    "pl_PL",
+    "ps_AF",
+    "pt_XX",
+    "sv_SE",
+    "sw_KE",
+    "ta_IN",
+    "te_IN",
+    "th_TH",
+    "tl_XX",
+    "uk_UA",
+    "ur_PK",
+    "xh_ZA",
+    "gl_ES",
+    "sl_SI"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "language_codes": "ML50",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "src_lang": "en_XX",
+  "tgt_lang": "hi_IN",
+  "tokenizer_class": "MBart50Tokenizer",
+  "unk_token": "<unk>"
+}

models/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9013c117330a5e2e1042c93ca678d33d3f6c2afa498e8a5c8079ab49db2ccd69
+size 6033

requirements.txt

@@ -0,0 +1,10 @@
+torch
+transformers
+datasets
+sentencepiece
+sacremoses
+ctranslate2
+gradio
+pandas
+scikit-learn
+accelerate

src/data/test.csv

@@ -0,0 +1,16 @@
+source,target,lang
+nadu,நாடு,ta
+tamil,தமிழ்,ta
+irukkeenga,இருக்கிறீர்கள்,ta
+khabar,খাবার,bn
+rahe,रहे,hi
+neengal,நீங்கள்,ta
+ho,हो,hi
+nadu,நாடு,ta
+bharat,भारत,hi
+desh,দেশ,bn
+vanakkam,வணக்கம்,ta
+achen,আছেন,bn
+kya,क्या,hi
+kar,कर,hi
+desh,দেশ,bn

src/data/train.csv

@@ -0,0 +1,121 @@
+source,target,lang
+tamil,தமிழ்,ta
+kya,क्या,hi
+aap,आप,hi
+apni,আপনি,bn
+amar,আমার,bn
+khabar,খাবার,bn
+bharat,भारत,hi
+apni,আপনি,bn
+bharat,भारत,hi
+vanakkam,வணக்கம்,ta
+en,என்,ta
+achen,আছেন,bn
+mera,मेरा,hi
+achen,আছেন,bn
+neengal,நீங்கள்,ta
+bharat,भारत,hi
+en,என்,ta
+sapadu,சாப்பாடு,ta
+rahe,रहे,hi
+hai,है,hi
+naam,नाम,hi
+namoshkar,নমস্কার,bn
+mera,मेरा,hi
+namoshkar,নমস্কার,bn
+aap,आप,hi
+kar,कर,hi
+jal,জল,bn
+rahe,रहे,hi
+eppadi,எப்படி,ta
+vanakkam,வணக்கம்,ta
+kar,कर,hi
+khabar,খাবার,bn
+tamil,தமிழ்,ta
+kemon,কেমন,bn
+jal,জল,bn
+thanni,தண்ணீர்,ta
+en,என்,ta
+kya,क्या,hi
+eppadi,எப்படி,ta
+khabar,খাবার,bn
+vanakkam,வணக்கம்,ta
+namaste,नमस्ते,hi
+desh,দেশ,bn
+thanni,தண்ணீர்,ta
+bangla,বাংলা,bn
+mera,मेरा,hi
+apni,আপনি,bn
+mera,मेरा,hi
+achen,আছেন,bn
+nam,নাম,bn
+irukkeenga,இருக்கிறீர்கள்,ta
+namoshkar,নমস্কার,bn
+desh,দেশ,bn
+mera,मेरा,hi
+nadu,நாடு,ta
+kar,कर,hi
+desh,দেশ,bn
+ho,हो,hi
+nam,নাম,bn
+rahe,रहे,hi
+rahe,रहे,hi
+bangla,বাংলা,bn
+apni,আপনি,bn
+naam,नाम,hi
+eppadi,எப்படி,ta
+namoshkar,নমস্কার,bn
+thanni,தண்ணீர்,ta
+eppadi,எப்படி,ta
+peyar,பெயர்,ta
+peyar,பெயர்,ta
+kar,कर,hi
+amar,আমার,bn
+thanni,தண்ணீர்,ta
+naam,नाम,hi
+kemon,কেমন,bn
+neengal,நீங்கள்,ta
+irukkeenga,இருக்கிறீர்கள்,ta
+bangla,বাংলা,bn
+en,என்,ta
+bangla,বাংলা,bn
+ho,हो,hi
+hai,है,hi
+nadu,நாடு,ta
+irukkeenga,இருக்கிறீர்கள்,ta
+tamil,தமிழ்,ta
+namaste,नमस्ते,hi
+vanakkam,வணக்கம்,ta
+naam,नाम,hi
+eppadi,எப்படி,ta
+bharat,भारत,hi
+amar,আমার,bn
+ho,हो,hi
+jal,জল,bn
+aap,आप,hi
+sapadu,சாப்பாடு,ta
+peyar,பெயர்,ta
+aap,आप,hi
+kya,क्या,hi
+kemon,কেমন,bn
+kemon,কেমন,bn
+amar,আমার,bn
+peyar,பெயர்,ta
+namaste,नमस्ते,hi
+nam,নাম,bn
+kya,क्या,hi
+irukkeenga,இருக்கிறீர்கள்,ta
+jal,জল,bn
+amar,আমার,bn
+nadu,நாடு,ta
+tamil,தமிழ்,ta
+bangla,বাংলা,bn
+hai,है,hi
+namaste,नमस्ते,hi
+thanni,தண்ணீர்,ta
+neengal,நீங்கள்,ta
+aap,आप,hi
+nam,নাম,bn
+hai,है,hi
+jal,জল,bn
+nam,নাম,bn

src/data/val.csv

@@ -0,0 +1,16 @@
+source,target,lang
+khabar,খাবার,bn
+kemon,কেমন,bn
+namoshkar,নমস্কার,bn
+sapadu,சாப்பாடு,ta
+sapadu,சாப்பாடு,ta
+namaste,नमस्ते,hi
+hai,है,hi
+neengal,நீங்கள்,ta
+apni,আপনি,bn
+peyar,பெயர்,ta
+en,என்,ta
+ho,हो,hi
+sapadu,சாப்பாடு,ta
+naam,नाम,hi
+achen,আছেন,bn

src/optimize.py

@@ -0,0 +1,101 @@
+import os
+import time
+import ctranslate2
+import transformers
+from datasets import load_dataset
+import pandas as pd
+
+MODEL_DIR = "models"
+CT2_MODEL_DIR = "models" # Set to models for HF Spaces compatibility (outputs model.bin here)
+
+def optimize_model():
+    print("Converting model to CTranslate2 format...")
+    # Ensure source files exist
+    if not any(f for f in os.listdir(MODEL_DIR) if f.startswith("pytorch_model") or f.endswith(".safetensors")):
+        print(f"Error: No source weights found in {MODEL_DIR}. Cannot convert.")
+        return
+
+    # Converter for mBART
+    converter = ctranslate2.converters.TransformersConverter(
+        MODEL_DIR, 
+        activation_scales=None,
+        copy_files=["tokenizer.json", "sentencepiece.bpe.model"] # Ensure tokenizer files are copied
+    )
+    
+    # Quantization often helps speed. Int8 is common.
+    converter.convert(
+        CT2_MODEL_DIR, 
+        quantization="int8",
+        force=True
+    )
+    print(f"Model converted and saved to {CT2_MODEL_DIR}")
+
+def benchmark():
+    print("\nStarting Benchmarking...")
+    
+    # Load original model (for size check only, inference might be slow to load)
+    # original_size = get_dir_size(MODEL_DIR)
+    # ct2_size = get_dir_size(CT2_MODEL_DIR)
+    # print(f"Original Model Size: {original_size / 1e6:.2f} MB")
+    # print(f"Optimized Model Size: {ct2_size / 1e6:.2f} MB")
+    
+    # Load CT2 model
+    translator = ctranslate2.Translator(CT2_MODEL_DIR)
+    tokenizer = transformers.MBart50TokenizerFast.from_pretrained(MODEL_DIR)
+    
+    # Test data
+    texts = ["Namaste", "Hello", "How are you", "Good morning", "India"]
+    target_lang = "hi_IN" # Test with Hindi
+    
+    tokenizer.src_lang = "en_XX"
+    
+    start_time = time.time()
+    
+    # Tokenize
+    source = tokenizer(texts, return_tensors="pt", padding=True)
+    input_tokens = [tokenizer.convert_ids_to_tokens(ids) for ids in source["input_ids"]]
+    
+    # Remove padding/eos if needed specifically for CT2, but usually it handles list of strings
+    # Actually CT2 expects list of list of str tokens
+    # Let's re-do properly for CT2 text input
+    
+    input_tokens_batch = []
+    for text in texts:
+        tokens = tokenizer.tokenize(text)
+        input_tokens_batch.append(tokens)
+        
+    # Translate
+    results = translator.translate_batch(
+        input_tokens_batch,
+        target_prefix=[[target_lang]] * len(texts) # Force target lang
+    )
+    
+    end_time = time.time()
+    
+    decoded = []
+    for result in results:
+        decoded.append(tokenizer.decode(tokenizer.convert_tokens_to_ids(result.hypotheses[0])))
+        
+    duration = end_time - start_time
+    print(f"Inference Time for {len(texts)} sentences: {duration:.4f}s")
+    print(f"Speed: {len(texts)/duration:.2f} sentences/s")
+    
+    for src, tgt in zip(texts, decoded):
+        print(f"{src} -> {tgt}")
+
+def get_dir_size(path):
+    total = 0
+    with os.scandir(path) as it:
+        for entry in it:
+            if entry.is_file():
+                total += entry.stat().st_size
+            elif entry.is_dir():
+                total += get_dir_size(entry.path)
+    return total
+
+if __name__ == "__main__":
+    if not os.path.exists(MODEL_DIR):
+        print(f"Model directory {MODEL_DIR} not found. Please train first.")
+    else:
+        optimize_model()
+        benchmark()

src/prepare_data.py

@@ -0,0 +1,74 @@
+import pandas as pd
+import os
+import random
+
+def create_dummy_data():
+    """Generates dummy transliteration data for Hindi, Bengali, and Tamil."""
+    
+    # Minimal dummy dataset
+    data = [
+        # Hindi
+        ("namaste", "नमस्ते", "hi"),
+        ("aap", "आप", "hi"),
+        ("kya", "क्या", "hi"),
+        ("kar", "कर", "hi"),
+        ("rahe", "रहे", "hi"),
+        ("ho", "हो", "hi"),
+        ("mera", "मेरा", "hi"),
+        ("naam", "नाम", "hi"),
+        ("hai", "है", "hi"),
+        ("bharat", "भारत", "hi"),
+        
+        # Bengali
+        ("namoshkar", "নমস্কার", "bn"),
+        ("apni", "আপনি", "bn"),
+        ("kemon", "কেমন", "bn"),
+        ("achen", "আছেন", "bn"),
+        ("amar", "আমার", "bn"),
+        ("nam", "নাম", "bn"),
+        ("bangla", "বাংলা", "bn"),
+        ("desh", "দেশ", "bn"),
+        ("khabar", "খাবার", "bn"),
+        ("jal", "জল", "bn"),
+
+        # Tamil
+        ("vanakkam", "வணக்கம்", "ta"),
+        ("neengal", "நீங்கள்", "ta"),
+        ("eppadi", "எப்படி", "ta"),
+        ("irukkeenga", "இருக்கிறீர்கள்", "ta"),
+        ("en", "என்", "ta"),
+        ("peyar", "பெயர்", "ta"),
+        ("tamil", "தமிழ்", "ta"),
+        ("nadu", "நாடு", "ta"),
+        ("sapadu", "சாப்பாடு", "ta"),
+        ("thanni", "தண்ணீர்", "ta")
+    ]
+
+    # Expand data slightly by duplicating to simulate a larger set for split
+    data = data * 5 
+    random.shuffle(data)
+
+    df = pd.DataFrame(data, columns=["source", "target", "lang"])
+
+    # Split into train, val, test (80-10-10)
+    train_size = int(0.8 * len(df))
+    val_size = int(0.1 * len(df))
+    
+    train_df = df[:train_size]
+    val_df = df[train_size:train_size+val_size]
+    test_df = df[train_size+val_size:]
+
+    output_dir = "data"
+    os.makedirs(output_dir, exist_ok=True)
+
+    train_df.to_csv(os.path.join(output_dir, "train.csv"), index=False)
+    val_df.to_csv(os.path.join(output_dir, "val.csv"), index=False)
+    test_df.to_csv(os.path.join(output_dir, "test.csv"), index=False)
+
+    print(f"Data generation complete.")
+    print(f"Train size: {len(train_df)}")
+    print(f"Val size: {len(val_df)}")
+    print(f"Test size: {len(test_df)}")
+
+if __name__ == "__main__":
+    create_dummy_data()

src/train.py

@@ -0,0 +1,141 @@
+import os
+import pandas as pd
+import torch
+from datasets import Dataset, DatasetDict
+from transformers import (
+    MBartForConditionalGeneration,
+    MBart50TokenizerFast,
+    Seq2SeqTrainingArguments,
+    Seq2SeqTrainer,
+    DataCollatorForSeq2Seq,
+)
+
+# ======================
+# CONFIG
+# ======================
+MODEL_NAME = "facebook/mbart-large-50-many-to-many-mmt"
+OUTPUT_DIR = "models/mbart-transliteration"
+
+MAX_INPUT_LENGTH = 128
+MAX_TARGET_LENGTH = 128
+
+BATCH_SIZE = 4          # CPU-safe
+EPOCHS = 1              # Increase later
+LEARNING_RATE = 5e-5
+
+SRC_LANG = "en_XX"
+TGT_LANG = "hi_IN"      # Hindi
+
+# ======================
+# LOAD DATA
+# ======================
+def load_data():
+    data_files = {
+        "train": "data/train.csv",
+        "validation": "data/val.csv",
+        "test": "data/test.csv",
+    }
+
+    dataset_dict = {}
+    for split, path in data_files.items():
+        df = pd.read_csv(path)
+
+        # REQUIRED columns
+        assert "source" in df.columns
+        assert "target" in df.columns
+
+        dataset_dict[split] = Dataset.from_pandas(df)
+
+    return DatasetDict(dataset_dict)
+
+# ======================
+# PREPROCESS (✅ FIXED)
+# ======================
+def preprocess_function(examples):
+    # ✅ MUST set every call (critical for mBART)
+    tokenizer.src_lang = SRC_LANG
+    tokenizer.tgt_lang = TGT_LANG
+
+    inputs = examples["source"]
+    targets = examples["target"]
+
+    model_inputs = tokenizer(
+        inputs,
+        max_length=MAX_INPUT_LENGTH,
+        truncation=True,
+        padding="max_length",
+    )
+
+    labels = tokenizer(
+        text_target=targets,
+        max_length=MAX_TARGET_LENGTH,
+        truncation=True,
+        padding="max_length",
+    )
+
+    model_inputs["labels"] = labels["input_ids"]
+    return model_inputs
+
+# ======================
+# TRAIN
+# ======================
+def main():
+    print("Loading tokenizer and model...")
+    global tokenizer
+
+    tokenizer = MBart50TokenizerFast.from_pretrained(MODEL_NAME)
+    model = MBartForConditionalGeneration.from_pretrained(MODEL_NAME, low_cpu_mem_usage=True)
+
+    print("Loading datasets...")
+    raw_datasets = load_data()
+
+    print("Tokenizing datasets...")
+    tokenized_datasets = raw_datasets.map(
+        preprocess_function,
+        batched=True,
+        remove_columns=raw_datasets["train"].column_names,
+    )
+
+    data_collator = DataCollatorForSeq2Seq(
+        tokenizer=tokenizer,
+        model=model,
+    )
+
+    training_args = Seq2SeqTrainingArguments(
+        output_dir=OUTPUT_DIR,
+        eval_strategy="epoch",
+        learning_rate=LEARNING_RATE,
+        per_device_train_batch_size=BATCH_SIZE,
+        per_device_eval_batch_size=BATCH_SIZE,
+        num_train_epochs=EPOCHS,
+        weight_decay=0.01,
+        save_total_limit=1,
+        save_strategy="epoch",
+        predict_with_generate=True,
+        logging_steps=10,
+        load_best_model_at_end=True,
+        report_to="none",
+        fp16=False,          # CPU safe
+    )
+
+    trainer = Seq2SeqTrainer(
+        model=model,
+        args=training_args,
+        train_dataset=tokenized_datasets["train"],
+        eval_dataset=tokenized_datasets["validation"],
+        tokenizer=tokenizer,
+        data_collator=data_collator,
+    )
+
+    print("Training started...")
+    trainer.train()
+
+    print("Saving model...")
+    trainer.save_model(OUTPUT_DIR)
+    tokenizer.save_pretrained(OUTPUT_DIR)
+
+    print(f"Training complete. Model saved to `{OUTPUT_DIR}`")
+
+# ======================
+if __name__ == "__main__":
+    main()