Initial release of Meitei-RoBERTa-Base (Bengali Script) with training logs and model card.

README.md

@@ -0,0 +1,62 @@
+
+---
+model_id: Badnyal/Meitei-RoBERTa
+language: "mni" # ISO 639-2 code for Manipuri (Meitei)
+tags: 
+- masked-language-modeling
+- transformer
+- roberta
+- meitei
+- low-resource
+- monolingual
+license: "cc-by-sa-4.0"
+---
+
+# Meitei-RoBERTa-Base (Bengali Script)
+
+This is a **monolingual RoBERTa Base model** pre-trained from scratch on the entire **Meitei Monolingual Corpus** (MWirelabs/meitei-monolingual-corpus). 
+It establishes the **State-of-the-Art foundational language model** for Meitei (Manipuri) in the Bengali script by leveraging a custom tokenizer and robust RoBERTa training objectives.
+
+## 💾 Model Details
+
+| Detail | Value |
+| :--- | :--- |
+| **Architecture** | RoBERTa Base (12 Layers, 12 Heads, 768 Hidden) |
+| **Total Parameters** | 126,031,648 |
+| **Pre-training Task** | Masked Language Modeling (MLM) |
+| **Tokenizer** | Byte-Level BPE |
+| **Vocabulary Size** | 52000 |
+| **Context Length** | 512 |
+
+## ⚙️ Training Information
+
+| Parameter | Value |
+| :--- | :--- |
+| **Dataset** | MWirelabs/meitei-monolingual-corpus (train split) |
+| **Data Size (Chunks)** | 353,123 blocks of 512 tokens |
+| **Effective Batch Size** | 256 |
+| **Learning Rate** | 0.0006 |
+| **Epochs** | 3 |
+| **Final Loss** | 4.18550 |
+
+## 📈 Loss Curve (For Visual Chart)
+
+The `training_loss_history.csv` file, included in this repository, contains the sequential loss data over training steps, allowing users to plot the convergence curve.
+
+## 💡 How to Use
+
+```python
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+# The repository ID is used to load both model and tokenizer
+REPO_ID = "Badnyal/Meitei-RoBERTa" 
+
+tokenizer = AutoTokenizer.from_pretrained(REPO_ID)
+model = AutoModelForMaskedLM.from_pretrained(REPO_ID)
+
+# Example: Tokenizing a Meitei sentence
+text = "ꯃꯤꯑꯣꯏꯕꯥ ꯑꯃꯥ ꯊꯤꯡꯖꯤꯟꯕꯥ ꯇꯥꯕꯥ꯫ ꯈꯣꯡꯎꯞ ꯄꯤꯕꯥ ꯇꯥꯕꯥ꯫" 
+inputs = tokenizer(text, return_tensors="pt")
+# The model can now be used for feature extraction, zero-shot inference, 
+# or fine-tuned for downstream tasks like Named Entity Recognition (NER) or classification.
+```

checkpoint-1380/config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "eos_token_id": 1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 3,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.57.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 52000
+}

checkpoint-1380/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8af0f5869472238d4377c634b3ffce4302a2ccfbc3b69c79f0810e7cbb895e0d
+size 504150808

checkpoint-1380/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a60354a24d1df0c6df3b4a8e446ff4f31d51bd0b46f63089079b6aef2b8eeb89
+size 1008422603

checkpoint-1380/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4bdb8c0daa13c0e6fd1faeb8d0cd13cf72d338a3572436eab98a1683ff226cc
+size 14645

checkpoint-1380/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20f288acd21e6afcbe6ed0dbfc211ea448b3d5dc72c8727ed48b0d623bdb674c
+size 1383

checkpoint-1380/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd8947e9bce174e8553636e8f6971f077f77a936b51b52cdd7070a5104f944ce
+size 1465

checkpoint-1380/special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1380/tokenizer_config.json

@@ -0,0 +1,53 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}

checkpoint-1380/trainer_state.json

@@ -0,0 +1,48 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1380,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.3624501631025734,
+      "grad_norm": 0.4742291271686554,
+      "learning_rate": 2.9939999999999998e-05,
+      "loss": 6.4565,
+      "step": 500
+    },
+    {
+      "epoch": 0.7249003262051468,
+      "grad_norm": 0.5359567403793335,
+      "learning_rate": 5.994e-05,
+      "loss": 4.3201,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 4140,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.296697231281357e+16,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1380/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f7549eb3763a9c4d7cf3ff0edbfd336c1ffe1406f78df5cf66043df3cc176d3
+size 5777

checkpoint-2760/config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "eos_token_id": 1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 3,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.57.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 52000
+}

checkpoint-2760/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bed97d14a18f7ace7e9aec9afefdefa85534f77d841b11ca5ae070eb5110ec7e
+size 504150808

checkpoint-2760/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e13e016912ff159d50148a09ee6c18281d6259cb98424ce6f62dc0f17c53eaf9
+size 1008422603

checkpoint-2760/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9e4077dba68b8bcdeb028955aafe1db7a8f256e51e622a2105cc82971423ff7
+size 14645

checkpoint-2760/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ec897a068d90bd14cd2b4ebbf344be41c3f69d39b2d7eb870c189a7308eaf18
+size 1383

checkpoint-2760/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08586d77a1761a3a1ec1920d4c0c81326d13398b5c337c4a0c66b11781ed38f8
+size 1465

checkpoint-2760/special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-2760/tokenizer_config.json

@@ -0,0 +1,53 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}

checkpoint-2760/trainer_state.json

@@ -0,0 +1,69 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 2760,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.3624501631025734,
+      "grad_norm": 0.4742291271686554,
+      "learning_rate": 2.9939999999999998e-05,
+      "loss": 6.4565,
+      "step": 500
+    },
+    {
+      "epoch": 0.7249003262051468,
+      "grad_norm": 0.5359567403793335,
+      "learning_rate": 5.994e-05,
+      "loss": 4.3201,
+      "step": 1000
+    },
+    {
+      "epoch": 1.0869880391446176,
+      "grad_norm": 0.4428270161151886,
+      "learning_rate": 8.994e-05,
+      "loss": 4.2592,
+      "step": 1500
+    },
+    {
+      "epoch": 1.449438202247191,
+      "grad_norm": 0.4446365237236023,
+      "learning_rate": 0.00011993999999999998,
+      "loss": 4.2331,
+      "step": 2000
+    },
+    {
+      "epoch": 1.8118883653497644,
+      "grad_norm": 0.7092395424842834,
+      "learning_rate": 0.00014994,
+      "loss": 4.2168,
+      "step": 2500
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 4140,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.8593394462562714e+17,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2760/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f7549eb3763a9c4d7cf3ff0edbfd336c1ffe1406f78df5cf66043df3cc176d3
+size 5777

checkpoint-4140/config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "eos_token_id": 1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 3,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.57.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 52000
+}

checkpoint-4140/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34cf01a40bc5231ecaa9bfa4063af838520f8fe60cea00e5b89c65c22069fb18
+size 504150808

checkpoint-4140/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f73c93edeec24497db533fd9ccbac7ecdf724d6cbca38c80ceca4956d02c0f0
+size 1008422603

checkpoint-4140/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ed945a78bb0c2401d9f48d0b285919b4b5118bad5e82a7f1963cd763f67f710
+size 14645

checkpoint-4140/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37c28d86d87229869f85f25e0381969a9d63089996702db9efd08cbb3ec61591
+size 1383

checkpoint-4140/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14b8e841653051ccc7faf4c2062bb12f15b917ad76a00cb9241f9f0029123179
+size 1465

checkpoint-4140/special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-4140/tokenizer_config.json

@@ -0,0 +1,53 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}

checkpoint-4140/trainer_state.json

@@ -0,0 +1,90 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 4140,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.3624501631025734,
+      "grad_norm": 0.4742291271686554,
+      "learning_rate": 2.9939999999999998e-05,
+      "loss": 6.4565,
+      "step": 500
+    },
+    {
+      "epoch": 0.7249003262051468,
+      "grad_norm": 0.5359567403793335,
+      "learning_rate": 5.994e-05,
+      "loss": 4.3201,
+      "step": 1000
+    },
+    {
+      "epoch": 1.0869880391446176,
+      "grad_norm": 0.4428270161151886,
+      "learning_rate": 8.994e-05,
+      "loss": 4.2592,
+      "step": 1500
+    },
+    {
+      "epoch": 1.449438202247191,
+      "grad_norm": 0.4446365237236023,
+      "learning_rate": 0.00011993999999999998,
+      "loss": 4.2331,
+      "step": 2000
+    },
+    {
+      "epoch": 1.8118883653497644,
+      "grad_norm": 0.7092395424842834,
+      "learning_rate": 0.00014994,
+      "loss": 4.2168,
+      "step": 2500
+    },
+    {
+      "epoch": 2.1739760782892352,
+      "grad_norm": 0.3681005835533142,
+      "learning_rate": 0.00017994,
+      "loss": 4.2023,
+      "step": 3000
+    },
+    {
+      "epoch": 2.5364262413918084,
+      "grad_norm": 0.5603296756744385,
+      "learning_rate": 0.00020993999999999999,
+      "loss": 4.1945,
+      "step": 3500
+    },
+    {
+      "epoch": 2.898876404494382,
+      "grad_norm": 0.34806573390960693,
+      "learning_rate": 0.00023993999999999996,
+      "loss": 4.1855,
+      "step": 4000
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 4140,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.789009169384407e+17,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-4140/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f7549eb3763a9c4d7cf3ff0edbfd336c1ffe1406f78df5cf66043df3cc176d3
+size 5777

config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "eos_token_id": 1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 3,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.57.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 52000
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34cf01a40bc5231ecaa9bfa4063af838520f8fe60cea00e5b89c65c22069fb18
+size 504150808

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,53 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f7549eb3763a9c4d7cf3ff0edbfd336c1ffe1406f78df5cf66043df3cc176d3
+size 5777

training_args.json

@@ -0,0 +1,148 @@
+{
+  "output_dir": "Meitei-RoBERTa-Base-MLM",
+  "overwrite_output_dir": true,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "no",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 32,
+  "per_device_eval_batch_size": 8,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 8,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 0.0006,
+  "weight_decay": 0.01,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 3,
+  "max_steps": -1,
+  "lr_scheduler_type": "linear",
+  "lr_scheduler_kwargs": {},
+  "warmup_ratio": 0.0,
+  "warmup_steps": 10000,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "./logs",
+  "logging_strategy": "steps",
+  "logging_first_step": false,
+  "logging_steps": 500,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "epoch",
+  "save_steps": 500,
+  "save_total_limit": 3,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "seed": 42,
+  "data_seed": null,
+  "jit_mode_eval": false,
+  "bf16": false,
+  "fp16": true,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "ddp_backend": null,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": [],
+  "dataloader_drop_last": false,
+  "eval_steps": null,
+  "dataloader_num_workers": 0,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": null,
+  "disable_tqdm": false,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": null,
+  "greater_is_better": null,
+  "ignore_data_skip": false,
+  "fsdp": [],
+  "fsdp_min_num_params": 0,
+  "fsdp_config": {
+    "min_num_params": 0,
+    "xla": false,
+    "xla_fsdp_v2": false,
+    "xla_fsdp_grad_ckpt": false
+  },
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "split_batches": false,
+    "dispatch_batches": null,
+    "even_batches": true,
+    "use_seedable_sampler": true,
+    "non_blocking": false,
+    "gradient_accumulation_kwargs": null
+  },
+  "parallelism_config": null,
+  "deepspeed": null,
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "tensorboard"
+  ],
+  "project": "huggingface",
+  "trackio_space_id": "trackio",
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_token": "<HUB_TOKEN>",
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "hub_revision": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": "<PUSH_TO_HUB_TOKEN>",
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "ddp_timeout": 1800,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": "no",
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": true
+}

training_loss_history.csv

@@ -0,0 +1,9 @@
+loss,step
+6.4565,500
+4.3201,1000
+4.2592,1500
+4.2331,2000
+4.2168,2500
+4.2023,3000
+4.1945,3500
+4.1855,4000