Uploading checkpoint-34000 for xlm-r - mar-deva

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,67 @@
+---
+library_name: transformers
+license: mit
+base_model: FacebookAI/xlm-roberta-base
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: mar-Deva
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mar-Deva
+
+This model is a fine-tuned version of [FacebookAI/xlm-roberta-base](https://huggingface.co/FacebookAI/xlm-roberta-base) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.2784
+- Accuracy: 0.7564
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- training_steps: 100000
+
+
+    ### Citation Information
+
+    If you use this model in your work, please cite the following paper. Additionally, if you require more details on training and performance, refer to the paper:
+
+    @misc{gurgurov2025smallmodelsbigimpact,
+        title={Small Models, Big Impact: Efficient Corpus and Graph-Based Adaptation of Small Multilingual Language Models for Low-Resource Languages}, 
+        author={Daniil Gurgurov and Ivan Vykopal and Josef van Genabith and Simon Ostermann},
+        year={2025},
+        eprint={2502.10140},
+        archivePrefix={arXiv},
+        primaryClass={cs.CL},
+        url={https://arxiv.org/abs/2502.10140}, 
+    }
+    

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "FacebookAI/xlm-roberta-base",
+  "adapters": {
+    "adapters": {},
+    "config_map": {},
+    "fusion_config_map": {},
+    "fusions": {}
+  },
+  "architectures": [
+    "XLMRobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.45.2",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250002
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16a29e7c415b659f124016770adbabd94d3c6f72025beefb8fecd9f8c83bc23b
+size 1113205088

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8a1653f621af66c5bca6d0e127ea6ac71e8c492d87ea636952a6611fef349c2
+size 2226530490

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a3d0b601c99d0ad7b602840c3fa99d29278e10117278812c239936e201988ff
+size 14512

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48ca79f1f2b9be6730c615a85f85545855ba91f5ef1d1d4b52116508ffe270ee
+size 14512

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd6eb79c156c6279b22b9f451323036b82317aa26779255918ce26cd1d77cd23
+size 1064

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8373f9cd3d27591e1924426bcc1c8799bc5a9affc4fc857982c5d66668dd1f41
+size 17082832

tokenizer_config.json

@@ -0,0 +1,54 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}

trainer_state.json

@@ -0,0 +1,1121 @@
+{
+  "best_metric": 1.276480793952942,
+  "best_model_checkpoint": "./model_fine-tune/glot/xlm-r/mar-Deva/checkpoint-34000",
+  "epoch": 79.81220657276995,
+  "eval_steps": 500,
+  "global_step": 34000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.1737089201877935,
+      "grad_norm": 4.907346725463867,
+      "learning_rate": 9.95e-05,
+      "loss": 1.7617,
+      "step": 500
+    },
+    {
+      "epoch": 1.1737089201877935,
+      "eval_accuracy": 0.682377250368625,
+      "eval_loss": 1.641917109489441,
+      "eval_runtime": 221.4281,
+      "eval_samples_per_second": 79.854,
+      "eval_steps_per_second": 2.497,
+      "step": 500
+    },
+    {
+      "epoch": 2.347417840375587,
+      "grad_norm": 3.960118293762207,
+      "learning_rate": 9.900000000000001e-05,
+      "loss": 1.6383,
+      "step": 1000
+    },
+    {
+      "epoch": 2.347417840375587,
+      "eval_accuracy": 0.6936361187270976,
+      "eval_loss": 1.5997333526611328,
+      "eval_runtime": 261.8177,
+      "eval_samples_per_second": 67.536,
+      "eval_steps_per_second": 2.112,
+      "step": 1000
+    },
+    {
+      "epoch": 3.52112676056338,
+      "grad_norm": 3.8852615356445312,
+      "learning_rate": 9.850000000000001e-05,
+      "loss": 1.5761,
+      "step": 1500
+    },
+    {
+      "epoch": 3.52112676056338,
+      "eval_accuracy": 0.6995766746280154,
+      "eval_loss": 1.5579007863998413,
+      "eval_runtime": 235.3567,
+      "eval_samples_per_second": 75.129,
+      "eval_steps_per_second": 2.35,
+      "step": 1500
+    },
+    {
+      "epoch": 4.694835680751174,
+      "grad_norm": 3.7597334384918213,
+      "learning_rate": 9.8e-05,
+      "loss": 1.5146,
+      "step": 2000
+    },
+    {
+      "epoch": 4.694835680751174,
+      "eval_accuracy": 0.7055014376140285,
+      "eval_loss": 1.5151420831680298,
+      "eval_runtime": 220.9997,
+      "eval_samples_per_second": 80.009,
+      "eval_steps_per_second": 2.502,
+      "step": 2000
+    },
+    {
+      "epoch": 5.868544600938967,
+      "grad_norm": 3.6979405879974365,
+      "learning_rate": 9.75e-05,
+      "loss": 1.4787,
+      "step": 2500
+    },
+    {
+      "epoch": 5.868544600938967,
+      "eval_accuracy": 0.7097442196297902,
+      "eval_loss": 1.4974758625030518,
+      "eval_runtime": 227.3225,
+      "eval_samples_per_second": 77.784,
+      "eval_steps_per_second": 2.433,
+      "step": 2500
+    },
+    {
+      "epoch": 7.042253521126761,
+      "grad_norm": 3.388044834136963,
+      "learning_rate": 9.7e-05,
+      "loss": 1.4325,
+      "step": 3000
+    },
+    {
+      "epoch": 7.042253521126761,
+      "eval_accuracy": 0.7124281101806074,
+      "eval_loss": 1.478604793548584,
+      "eval_runtime": 220.1541,
+      "eval_samples_per_second": 80.316,
+      "eval_steps_per_second": 2.512,
+      "step": 3000
+    },
+    {
+      "epoch": 8.215962441314554,
+      "grad_norm": 3.2289981842041016,
+      "learning_rate": 9.65e-05,
+      "loss": 1.4016,
+      "step": 3500
+    },
+    {
+      "epoch": 8.215962441314554,
+      "eval_accuracy": 0.716293484657873,
+      "eval_loss": 1.4516507387161255,
+      "eval_runtime": 246.0988,
+      "eval_samples_per_second": 71.849,
+      "eval_steps_per_second": 2.247,
+      "step": 3500
+    },
+    {
+      "epoch": 9.389671361502348,
+      "grad_norm": 3.4439873695373535,
+      "learning_rate": 9.6e-05,
+      "loss": 1.3756,
+      "step": 4000
+    },
+    {
+      "epoch": 9.389671361502348,
+      "eval_accuracy": 0.7190285665969247,
+      "eval_loss": 1.4359241724014282,
+      "eval_runtime": 217.7951,
+      "eval_samples_per_second": 81.186,
+      "eval_steps_per_second": 2.539,
+      "step": 4000
+    },
+    {
+      "epoch": 10.56338028169014,
+      "grad_norm": 3.6135294437408447,
+      "learning_rate": 9.55e-05,
+      "loss": 1.3469,
+      "step": 4500
+    },
+    {
+      "epoch": 10.56338028169014,
+      "eval_accuracy": 0.7210358714994849,
+      "eval_loss": 1.4336496591567993,
+      "eval_runtime": 235.5365,
+      "eval_samples_per_second": 75.071,
+      "eval_steps_per_second": 2.348,
+      "step": 4500
+    },
+    {
+      "epoch": 11.737089201877934,
+      "grad_norm": 3.4069502353668213,
+      "learning_rate": 9.5e-05,
+      "loss": 1.3302,
+      "step": 5000
+    },
+    {
+      "epoch": 11.737089201877934,
+      "eval_accuracy": 0.7228719713830046,
+      "eval_loss": 1.4151064157485962,
+      "eval_runtime": 219.1479,
+      "eval_samples_per_second": 80.685,
+      "eval_steps_per_second": 2.523,
+      "step": 5000
+    },
+    {
+      "epoch": 12.910798122065728,
+      "grad_norm": 3.4197683334350586,
+      "learning_rate": 9.449999999999999e-05,
+      "loss": 1.31,
+      "step": 5500
+    },
+    {
+      "epoch": 12.910798122065728,
+      "eval_accuracy": 0.7252864162262277,
+      "eval_loss": 1.404516339302063,
+      "eval_runtime": 224.6695,
+      "eval_samples_per_second": 78.702,
+      "eval_steps_per_second": 2.461,
+      "step": 5500
+    },
+    {
+      "epoch": 14.084507042253522,
+      "grad_norm": 3.5801749229431152,
+      "learning_rate": 9.4e-05,
+      "loss": 1.2846,
+      "step": 6000
+    },
+    {
+      "epoch": 14.084507042253522,
+      "eval_accuracy": 0.7258948694302616,
+      "eval_loss": 1.4096269607543945,
+      "eval_runtime": 258.3434,
+      "eval_samples_per_second": 68.444,
+      "eval_steps_per_second": 2.141,
+      "step": 6000
+    },
+    {
+      "epoch": 15.258215962441314,
+      "grad_norm": 3.388658285140991,
+      "learning_rate": 9.350000000000001e-05,
+      "loss": 1.2717,
+      "step": 6500
+    },
+    {
+      "epoch": 15.258215962441314,
+      "eval_accuracy": 0.7286868885121539,
+      "eval_loss": 1.393049716949463,
+      "eval_runtime": 217.5523,
+      "eval_samples_per_second": 81.277,
+      "eval_steps_per_second": 2.542,
+      "step": 6500
+    },
+    {
+      "epoch": 16.431924882629108,
+      "grad_norm": 3.4384610652923584,
+      "learning_rate": 9.300000000000001e-05,
+      "loss": 1.2576,
+      "step": 7000
+    },
+    {
+      "epoch": 16.431924882629108,
+      "eval_accuracy": 0.7280798394010247,
+      "eval_loss": 1.3944166898727417,
+      "eval_runtime": 218.7336,
+      "eval_samples_per_second": 80.838,
+      "eval_steps_per_second": 2.528,
+      "step": 7000
+    },
+    {
+      "epoch": 17.6056338028169,
+      "grad_norm": 5.340916156768799,
+      "learning_rate": 9.250000000000001e-05,
+      "loss": 1.2392,
+      "step": 7500
+    },
+    {
+      "epoch": 17.6056338028169,
+      "eval_accuracy": 0.7315341751407455,
+      "eval_loss": 1.3892707824707031,
+      "eval_runtime": 247.2216,
+      "eval_samples_per_second": 71.523,
+      "eval_steps_per_second": 2.237,
+      "step": 7500
+    },
+    {
+      "epoch": 18.779342723004696,
+      "grad_norm": 3.373849630355835,
+      "learning_rate": 9.200000000000001e-05,
+      "loss": 1.2219,
+      "step": 8000
+    },
+    {
+      "epoch": 18.779342723004696,
+      "eval_accuracy": 0.7337430185709165,
+      "eval_loss": 1.3687783479690552,
+      "eval_runtime": 223.4847,
+      "eval_samples_per_second": 79.12,
+      "eval_steps_per_second": 2.474,
+      "step": 8000
+    },
+    {
+      "epoch": 19.953051643192488,
+      "grad_norm": 3.577197551727295,
+      "learning_rate": 9.15e-05,
+      "loss": 1.2066,
+      "step": 8500
+    },
+    {
+      "epoch": 19.953051643192488,
+      "eval_accuracy": 0.7335215003148629,
+      "eval_loss": 1.3642122745513916,
+      "eval_runtime": 252.5227,
+      "eval_samples_per_second": 70.021,
+      "eval_steps_per_second": 2.19,
+      "step": 8500
+    },
+    {
+      "epoch": 21.12676056338028,
+      "grad_norm": 3.131434202194214,
+      "learning_rate": 9.1e-05,
+      "loss": 1.1949,
+      "step": 9000
+    },
+    {
+      "epoch": 21.12676056338028,
+      "eval_accuracy": 0.7358405303628205,
+      "eval_loss": 1.361973524093628,
+      "eval_runtime": 202.337,
+      "eval_samples_per_second": 87.389,
+      "eval_steps_per_second": 2.733,
+      "step": 9000
+    },
+    {
+      "epoch": 22.300469483568076,
+      "grad_norm": 3.1661245822906494,
+      "learning_rate": 9.05e-05,
+      "loss": 1.1777,
+      "step": 9500
+    },
+    {
+      "epoch": 22.300469483568076,
+      "eval_accuracy": 0.7363205030264911,
+      "eval_loss": 1.3682575225830078,
+      "eval_runtime": 201.3827,
+      "eval_samples_per_second": 87.803,
+      "eval_steps_per_second": 2.746,
+      "step": 9500
+    },
+    {
+      "epoch": 23.474178403755868,
+      "grad_norm": 3.1136245727539062,
+      "learning_rate": 9e-05,
+      "loss": 1.1664,
+      "step": 10000
+    },
+    {
+      "epoch": 23.474178403755868,
+      "eval_accuracy": 0.7363390704473007,
+      "eval_loss": 1.3535041809082031,
+      "eval_runtime": 204.0918,
+      "eval_samples_per_second": 86.637,
+      "eval_steps_per_second": 2.71,
+      "step": 10000
+    },
+    {
+      "epoch": 24.647887323943664,
+      "grad_norm": 3.609222173690796,
+      "learning_rate": 8.950000000000001e-05,
+      "loss": 1.1549,
+      "step": 10500
+    },
+    {
+      "epoch": 24.647887323943664,
+      "eval_accuracy": 0.7367611964598777,
+      "eval_loss": 1.3562211990356445,
+      "eval_runtime": 220.1403,
+      "eval_samples_per_second": 80.321,
+      "eval_steps_per_second": 2.512,
+      "step": 10500
+    },
+    {
+      "epoch": 25.821596244131456,
+      "grad_norm": 3.2088677883148193,
+      "learning_rate": 8.900000000000001e-05,
+      "loss": 1.1409,
+      "step": 11000
+    },
+    {
+      "epoch": 25.821596244131456,
+      "eval_accuracy": 0.7380471720029462,
+      "eval_loss": 1.3453235626220703,
+      "eval_runtime": 208.3196,
+      "eval_samples_per_second": 84.879,
+      "eval_steps_per_second": 2.655,
+      "step": 11000
+    },
+    {
+      "epoch": 26.995305164319248,
+      "grad_norm": 3.0804500579833984,
+      "learning_rate": 8.850000000000001e-05,
+      "loss": 1.1359,
+      "step": 11500
+    },
+    {
+      "epoch": 26.995305164319248,
+      "eval_accuracy": 0.7393776591908848,
+      "eval_loss": 1.3371671438217163,
+      "eval_runtime": 201.8856,
+      "eval_samples_per_second": 87.584,
+      "eval_steps_per_second": 2.739,
+      "step": 11500
+    },
+    {
+      "epoch": 28.169014084507044,
+      "grad_norm": 3.4229302406311035,
+      "learning_rate": 8.800000000000001e-05,
+      "loss": 1.1251,
+      "step": 12000
+    },
+    {
+      "epoch": 28.169014084507044,
+      "eval_accuracy": 0.7405217538157773,
+      "eval_loss": 1.3338407278060913,
+      "eval_runtime": 201.6981,
+      "eval_samples_per_second": 87.666,
+      "eval_steps_per_second": 2.742,
+      "step": 12000
+    },
+    {
+      "epoch": 29.342723004694836,
+      "grad_norm": 3.222292900085449,
+      "learning_rate": 8.75e-05,
+      "loss": 1.112,
+      "step": 12500
+    },
+    {
+      "epoch": 29.342723004694836,
+      "eval_accuracy": 0.741253660719189,
+      "eval_loss": 1.3335049152374268,
+      "eval_runtime": 200.2708,
+      "eval_samples_per_second": 88.29,
+      "eval_steps_per_second": 2.761,
+      "step": 12500
+    },
+    {
+      "epoch": 30.516431924882628,
+      "grad_norm": 3.8668177127838135,
+      "learning_rate": 8.7e-05,
+      "loss": 1.1001,
+      "step": 13000
+    },
+    {
+      "epoch": 30.516431924882628,
+      "eval_accuracy": 0.7424599044116853,
+      "eval_loss": 1.3257431983947754,
+      "eval_runtime": 201.872,
+      "eval_samples_per_second": 87.59,
+      "eval_steps_per_second": 2.739,
+      "step": 13000
+    },
+    {
+      "epoch": 31.690140845070424,
+      "grad_norm": 3.0373659133911133,
+      "learning_rate": 8.65e-05,
+      "loss": 1.0888,
+      "step": 13500
+    },
+    {
+      "epoch": 31.690140845070424,
+      "eval_accuracy": 0.7430678545716343,
+      "eval_loss": 1.323199987411499,
+      "eval_runtime": 202.3595,
+      "eval_samples_per_second": 87.379,
+      "eval_steps_per_second": 2.733,
+      "step": 13500
+    },
+    {
+      "epoch": 32.863849765258216,
+      "grad_norm": 3.3151516914367676,
+      "learning_rate": 8.6e-05,
+      "loss": 1.0876,
+      "step": 14000
+    },
+    {
+      "epoch": 32.863849765258216,
+      "eval_accuracy": 0.7430443796717692,
+      "eval_loss": 1.3175766468048096,
+      "eval_runtime": 217.5012,
+      "eval_samples_per_second": 81.296,
+      "eval_steps_per_second": 2.543,
+      "step": 14000
+    },
+    {
+      "epoch": 34.03755868544601,
+      "grad_norm": 2.909252643585205,
+      "learning_rate": 8.55e-05,
+      "loss": 1.0666,
+      "step": 14500
+    },
+    {
+      "epoch": 34.03755868544601,
+      "eval_accuracy": 0.7433313275369421,
+      "eval_loss": 1.333316683769226,
+      "eval_runtime": 201.0956,
+      "eval_samples_per_second": 87.928,
+      "eval_steps_per_second": 2.75,
+      "step": 14500
+    },
+    {
+      "epoch": 35.2112676056338,
+      "grad_norm": 3.8464670181274414,
+      "learning_rate": 8.5e-05,
+      "loss": 1.061,
+      "step": 15000
+    },
+    {
+      "epoch": 35.2112676056338,
+      "eval_accuracy": 0.7443912183198736,
+      "eval_loss": 1.3244580030441284,
+      "eval_runtime": 205.6215,
+      "eval_samples_per_second": 85.993,
+      "eval_steps_per_second": 2.689,
+      "step": 15000
+    },
+    {
+      "epoch": 36.3849765258216,
+      "grad_norm": 2.873279094696045,
+      "learning_rate": 8.450000000000001e-05,
+      "loss": 1.0505,
+      "step": 15500
+    },
+    {
+      "epoch": 36.3849765258216,
+      "eval_accuracy": 0.7435481707476262,
+      "eval_loss": 1.3140238523483276,
+      "eval_runtime": 213.2162,
+      "eval_samples_per_second": 82.93,
+      "eval_steps_per_second": 2.594,
+      "step": 15500
+    },
+    {
+      "epoch": 37.55868544600939,
+      "grad_norm": 3.3015851974487305,
+      "learning_rate": 8.4e-05,
+      "loss": 1.0419,
+      "step": 16000
+    },
+    {
+      "epoch": 37.55868544600939,
+      "eval_accuracy": 0.7448983633712033,
+      "eval_loss": 1.3211318254470825,
+      "eval_runtime": 219.2166,
+      "eval_samples_per_second": 80.66,
+      "eval_steps_per_second": 2.523,
+      "step": 16000
+    },
+    {
+      "epoch": 38.732394366197184,
+      "grad_norm": 3.0020763874053955,
+      "learning_rate": 8.35e-05,
+      "loss": 1.0364,
+      "step": 16500
+    },
+    {
+      "epoch": 38.732394366197184,
+      "eval_accuracy": 0.7462611325827592,
+      "eval_loss": 1.3081616163253784,
+      "eval_runtime": 211.9425,
+      "eval_samples_per_second": 83.428,
+      "eval_steps_per_second": 2.609,
+      "step": 16500
+    },
+    {
+      "epoch": 39.906103286384976,
+      "grad_norm": 3.087907552719116,
+      "learning_rate": 8.3e-05,
+      "loss": 1.0281,
+      "step": 17000
+    },
+    {
+      "epoch": 39.906103286384976,
+      "eval_accuracy": 0.746103364149513,
+      "eval_loss": 1.3020210266113281,
+      "eval_runtime": 204.4235,
+      "eval_samples_per_second": 86.497,
+      "eval_steps_per_second": 2.705,
+      "step": 17000
+    },
+    {
+      "epoch": 41.07981220657277,
+      "grad_norm": 2.9209628105163574,
+      "learning_rate": 8.25e-05,
+      "loss": 1.0213,
+      "step": 17500
+    },
+    {
+      "epoch": 41.07981220657277,
+      "eval_accuracy": 0.7478941463260773,
+      "eval_loss": 1.3000065088272095,
+      "eval_runtime": 205.342,
+      "eval_samples_per_second": 86.11,
+      "eval_steps_per_second": 2.693,
+      "step": 17500
+    },
+    {
+      "epoch": 42.25352112676056,
+      "grad_norm": 3.0207581520080566,
+      "learning_rate": 8.2e-05,
+      "loss": 1.0135,
+      "step": 18000
+    },
+    {
+      "epoch": 42.25352112676056,
+      "eval_accuracy": 0.7476731220604421,
+      "eval_loss": 1.3038578033447266,
+      "eval_runtime": 201.338,
+      "eval_samples_per_second": 87.822,
+      "eval_steps_per_second": 2.747,
+      "step": 18000
+    },
+    {
+      "epoch": 43.42723004694836,
+      "grad_norm": 3.0786807537078857,
+      "learning_rate": 8.15e-05,
+      "loss": 1.0053,
+      "step": 18500
+    },
+    {
+      "epoch": 43.42723004694836,
+      "eval_accuracy": 0.7473569751103342,
+      "eval_loss": 1.3108924627304077,
+      "eval_runtime": 200.3973,
+      "eval_samples_per_second": 88.235,
+      "eval_steps_per_second": 2.76,
+      "step": 18500
+    },
+    {
+      "epoch": 44.60093896713615,
+      "grad_norm": 2.8949594497680664,
+      "learning_rate": 8.1e-05,
+      "loss": 0.9974,
+      "step": 19000
+    },
+    {
+      "epoch": 44.60093896713615,
+      "eval_accuracy": 0.7478287545408583,
+      "eval_loss": 1.3078181743621826,
+      "eval_runtime": 199.5198,
+      "eval_samples_per_second": 88.623,
+      "eval_steps_per_second": 2.772,
+      "step": 19000
+    },
+    {
+      "epoch": 45.774647887323944,
+      "grad_norm": 3.058262348175049,
+      "learning_rate": 8.05e-05,
+      "loss": 0.9882,
+      "step": 19500
+    },
+    {
+      "epoch": 45.774647887323944,
+      "eval_accuracy": 0.7491708465969346,
+      "eval_loss": 1.2856029272079468,
+      "eval_runtime": 199.6673,
+      "eval_samples_per_second": 88.557,
+      "eval_steps_per_second": 2.77,
+      "step": 19500
+    },
+    {
+      "epoch": 46.948356807511736,
+      "grad_norm": 2.8529019355773926,
+      "learning_rate": 8e-05,
+      "loss": 0.9856,
+      "step": 20000
+    },
+    {
+      "epoch": 46.948356807511736,
+      "eval_accuracy": 0.7482058218213462,
+      "eval_loss": 1.3070999383926392,
+      "eval_runtime": 199.4786,
+      "eval_samples_per_second": 88.641,
+      "eval_steps_per_second": 2.772,
+      "step": 20000
+    },
+    {
+      "epoch": 48.12206572769953,
+      "grad_norm": 3.221163749694824,
+      "learning_rate": 7.950000000000001e-05,
+      "loss": 0.9758,
+      "step": 20500
+    },
+    {
+      "epoch": 48.12206572769953,
+      "eval_accuracy": 0.7491880441185212,
+      "eval_loss": 1.3054530620574951,
+      "eval_runtime": 200.7787,
+      "eval_samples_per_second": 88.067,
+      "eval_steps_per_second": 2.754,
+      "step": 20500
+    },
+    {
+      "epoch": 49.29577464788732,
+      "grad_norm": 2.8335845470428467,
+      "learning_rate": 7.900000000000001e-05,
+      "loss": 0.9691,
+      "step": 21000
+    },
+    {
+      "epoch": 49.29577464788732,
+      "eval_accuracy": 0.749970950525818,
+      "eval_loss": 1.2995696067810059,
+      "eval_runtime": 199.9883,
+      "eval_samples_per_second": 88.415,
+      "eval_steps_per_second": 2.765,
+      "step": 21000
+    },
+    {
+      "epoch": 50.46948356807512,
+      "grad_norm": 3.068582773208618,
+      "learning_rate": 7.850000000000001e-05,
+      "loss": 0.9593,
+      "step": 21500
+    },
+    {
+      "epoch": 50.46948356807512,
+      "eval_accuracy": 0.7508033170148307,
+      "eval_loss": 1.291608214378357,
+      "eval_runtime": 206.6584,
+      "eval_samples_per_second": 85.562,
+      "eval_steps_per_second": 2.676,
+      "step": 21500
+    },
+    {
+      "epoch": 51.64319248826291,
+      "grad_norm": 3.001044511795044,
+      "learning_rate": 7.800000000000001e-05,
+      "loss": 0.9592,
+      "step": 22000
+    },
+    {
+      "epoch": 51.64319248826291,
+      "eval_accuracy": 0.7509144022289539,
+      "eval_loss": 1.3004404306411743,
+      "eval_runtime": 200.0922,
+      "eval_samples_per_second": 88.369,
+      "eval_steps_per_second": 2.764,
+      "step": 22000
+    },
+    {
+      "epoch": 52.816901408450704,
+      "grad_norm": 2.9949467182159424,
+      "learning_rate": 7.75e-05,
+      "loss": 0.9451,
+      "step": 22500
+    },
+    {
+      "epoch": 52.816901408450704,
+      "eval_accuracy": 0.7508627912229723,
+      "eval_loss": 1.2902069091796875,
+      "eval_runtime": 202.9339,
+      "eval_samples_per_second": 87.132,
+      "eval_steps_per_second": 2.725,
+      "step": 22500
+    },
+    {
+      "epoch": 53.990610328638496,
+      "grad_norm": 2.9131195545196533,
+      "learning_rate": 7.7e-05,
+      "loss": 0.9418,
+      "step": 23000
+    },
+    {
+      "epoch": 53.990610328638496,
+      "eval_accuracy": 0.7521223540839408,
+      "eval_loss": 1.281449794769287,
+      "eval_runtime": 203.7276,
+      "eval_samples_per_second": 86.792,
+      "eval_steps_per_second": 2.714,
+      "step": 23000
+    },
+    {
+      "epoch": 55.16431924882629,
+      "grad_norm": 2.993138313293457,
+      "learning_rate": 7.65e-05,
+      "loss": 0.9368,
+      "step": 23500
+    },
+    {
+      "epoch": 55.16431924882629,
+      "eval_accuracy": 0.7515514473723303,
+      "eval_loss": 1.292709231376648,
+      "eval_runtime": 199.488,
+      "eval_samples_per_second": 88.637,
+      "eval_steps_per_second": 2.772,
+      "step": 23500
+    },
+    {
+      "epoch": 56.33802816901409,
+      "grad_norm": 3.2876174449920654,
+      "learning_rate": 7.6e-05,
+      "loss": 0.933,
+      "step": 24000
+    },
+    {
+      "epoch": 56.33802816901409,
+      "eval_accuracy": 0.7514791832510388,
+      "eval_loss": 1.3007817268371582,
+      "eval_runtime": 199.2802,
+      "eval_samples_per_second": 88.729,
+      "eval_steps_per_second": 2.775,
+      "step": 24000
+    },
+    {
+      "epoch": 57.51173708920188,
+      "grad_norm": 2.8598861694335938,
+      "learning_rate": 7.55e-05,
+      "loss": 0.9293,
+      "step": 24500
+    },
+    {
+      "epoch": 57.51173708920188,
+      "eval_accuracy": 0.7523163662895046,
+      "eval_loss": 1.2937757968902588,
+      "eval_runtime": 199.5861,
+      "eval_samples_per_second": 88.593,
+      "eval_steps_per_second": 2.771,
+      "step": 24500
+    },
+    {
+      "epoch": 58.68544600938967,
+      "grad_norm": 2.987820625305176,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 0.9127,
+      "step": 25000
+    },
+    {
+      "epoch": 58.68544600938967,
+      "eval_accuracy": 0.7522867644477332,
+      "eval_loss": 1.2981115579605103,
+      "eval_runtime": 215.9064,
+      "eval_samples_per_second": 81.897,
+      "eval_steps_per_second": 2.561,
+      "step": 25000
+    },
+    {
+      "epoch": 59.859154929577464,
+      "grad_norm": 3.0826175212860107,
+      "learning_rate": 7.450000000000001e-05,
+      "loss": 0.9163,
+      "step": 25500
+    },
+    {
+      "epoch": 59.859154929577464,
+      "eval_accuracy": 0.7524350211592462,
+      "eval_loss": 1.2923673391342163,
+      "eval_runtime": 215.5954,
+      "eval_samples_per_second": 82.015,
+      "eval_steps_per_second": 2.565,
+      "step": 25500
+    },
+    {
+      "epoch": 61.032863849765256,
+      "grad_norm": 3.045530080795288,
+      "learning_rate": 7.4e-05,
+      "loss": 0.9098,
+      "step": 26000
+    },
+    {
+      "epoch": 61.032863849765256,
+      "eval_accuracy": 0.7529520807730653,
+      "eval_loss": 1.3031412363052368,
+      "eval_runtime": 202.3207,
+      "eval_samples_per_second": 87.396,
+      "eval_steps_per_second": 2.733,
+      "step": 26000
+    },
+    {
+      "epoch": 62.20657276995305,
+      "grad_norm": 3.1517746448516846,
+      "learning_rate": 7.35e-05,
+      "loss": 0.9,
+      "step": 26500
+    },
+    {
+      "epoch": 62.20657276995305,
+      "eval_accuracy": 0.7522288481902238,
+      "eval_loss": 1.319172739982605,
+      "eval_runtime": 199.9694,
+      "eval_samples_per_second": 88.424,
+      "eval_steps_per_second": 2.765,
+      "step": 26500
+    },
+    {
+      "epoch": 63.38028169014085,
+      "grad_norm": 3.1550400257110596,
+      "learning_rate": 7.3e-05,
+      "loss": 0.8975,
+      "step": 27000
+    },
+    {
+      "epoch": 63.38028169014085,
+      "eval_accuracy": 0.754023096118599,
+      "eval_loss": 1.290266513824463,
+      "eval_runtime": 199.85,
+      "eval_samples_per_second": 88.476,
+      "eval_steps_per_second": 2.767,
+      "step": 27000
+    },
+    {
+      "epoch": 64.55399061032864,
+      "grad_norm": 2.9039342403411865,
+      "learning_rate": 7.25e-05,
+      "loss": 0.8885,
+      "step": 27500
+    },
+    {
+      "epoch": 64.55399061032864,
+      "eval_accuracy": 0.753646522940533,
+      "eval_loss": 1.2913925647735596,
+      "eval_runtime": 216.7264,
+      "eval_samples_per_second": 81.587,
+      "eval_steps_per_second": 2.552,
+      "step": 27500
+    },
+    {
+      "epoch": 65.72769953051643,
+      "grad_norm": 3.386866569519043,
+      "learning_rate": 7.2e-05,
+      "loss": 0.8882,
+      "step": 28000
+    },
+    {
+      "epoch": 65.72769953051643,
+      "eval_accuracy": 0.7540241100062917,
+      "eval_loss": 1.3034321069717407,
+      "eval_runtime": 198.4056,
+      "eval_samples_per_second": 89.12,
+      "eval_steps_per_second": 2.787,
+      "step": 28000
+    },
+    {
+      "epoch": 66.90140845070422,
+      "grad_norm": 3.0727860927581787,
+      "learning_rate": 7.15e-05,
+      "loss": 0.8776,
+      "step": 28500
+    },
+    {
+      "epoch": 66.90140845070422,
+      "eval_accuracy": 0.7545831279617436,
+      "eval_loss": 1.2843329906463623,
+      "eval_runtime": 201.974,
+      "eval_samples_per_second": 87.546,
+      "eval_steps_per_second": 2.738,
+      "step": 28500
+    },
+    {
+      "epoch": 68.07511737089202,
+      "grad_norm": 3.0493087768554688,
+      "learning_rate": 7.1e-05,
+      "loss": 0.8762,
+      "step": 29000
+    },
+    {
+      "epoch": 68.07511737089202,
+      "eval_accuracy": 0.755696390419431,
+      "eval_loss": 1.2895519733428955,
+      "eval_runtime": 201.8699,
+      "eval_samples_per_second": 87.591,
+      "eval_steps_per_second": 2.739,
+      "step": 29000
+    },
+    {
+      "epoch": 69.24882629107981,
+      "grad_norm": 3.3382585048675537,
+      "learning_rate": 7.05e-05,
+      "loss": 0.8668,
+      "step": 29500
+    },
+    {
+      "epoch": 69.24882629107981,
+      "eval_accuracy": 0.7549284378596087,
+      "eval_loss": 1.2944034337997437,
+      "eval_runtime": 200.0058,
+      "eval_samples_per_second": 88.407,
+      "eval_steps_per_second": 2.765,
+      "step": 29500
+    },
+    {
+      "epoch": 70.4225352112676,
+      "grad_norm": 2.944561243057251,
+      "learning_rate": 7e-05,
+      "loss": 0.8646,
+      "step": 30000
+    },
+    {
+      "epoch": 70.4225352112676,
+      "eval_accuracy": 0.7547729512512319,
+      "eval_loss": 1.3033416271209717,
+      "eval_runtime": 199.6135,
+      "eval_samples_per_second": 88.581,
+      "eval_steps_per_second": 2.77,
+      "step": 30000
+    },
+    {
+      "epoch": 71.59624413145539,
+      "grad_norm": 2.918184995651245,
+      "learning_rate": 6.95e-05,
+      "loss": 0.8545,
+      "step": 30500
+    },
+    {
+      "epoch": 71.59624413145539,
+      "eval_accuracy": 0.7549324708129699,
+      "eval_loss": 1.2862614393234253,
+      "eval_runtime": 198.4738,
+      "eval_samples_per_second": 89.09,
+      "eval_steps_per_second": 2.786,
+      "step": 30500
+    },
+    {
+      "epoch": 72.7699530516432,
+      "grad_norm": 3.2001590728759766,
+      "learning_rate": 6.9e-05,
+      "loss": 0.8546,
+      "step": 31000
+    },
+    {
+      "epoch": 72.7699530516432,
+      "eval_accuracy": 0.7545426996229516,
+      "eval_loss": 1.2923203706741333,
+      "eval_runtime": 199.1256,
+      "eval_samples_per_second": 88.798,
+      "eval_steps_per_second": 2.777,
+      "step": 31000
+    },
+    {
+      "epoch": 73.94366197183099,
+      "grad_norm": 3.18835711479187,
+      "learning_rate": 6.850000000000001e-05,
+      "loss": 0.8481,
+      "step": 31500
+    },
+    {
+      "epoch": 73.94366197183099,
+      "eval_accuracy": 0.7555989174331249,
+      "eval_loss": 1.3080549240112305,
+      "eval_runtime": 202.6015,
+      "eval_samples_per_second": 87.275,
+      "eval_steps_per_second": 2.729,
+      "step": 31500
+    },
+    {
+      "epoch": 75.11737089201878,
+      "grad_norm": 2.8794021606445312,
+      "learning_rate": 6.800000000000001e-05,
+      "loss": 0.847,
+      "step": 32000
+    },
+    {
+      "epoch": 75.11737089201878,
+      "eval_accuracy": 0.7562778395307137,
+      "eval_loss": 1.3006749153137207,
+      "eval_runtime": 204.7682,
+      "eval_samples_per_second": 86.351,
+      "eval_steps_per_second": 2.701,
+      "step": 32000
+    },
+    {
+      "epoch": 76.29107981220658,
+      "grad_norm": 3.278358221054077,
+      "learning_rate": 6.750000000000001e-05,
+      "loss": 0.8415,
+      "step": 32500
+    },
+    {
+      "epoch": 76.29107981220658,
+      "eval_accuracy": 0.7555797843893883,
+      "eval_loss": 1.2970653772354126,
+      "eval_runtime": 201.1856,
+      "eval_samples_per_second": 87.889,
+      "eval_steps_per_second": 2.749,
+      "step": 32500
+    },
+    {
+      "epoch": 77.46478873239437,
+      "grad_norm": 2.8185603618621826,
+      "learning_rate": 6.7e-05,
+      "loss": 0.8297,
+      "step": 33000
+    },
+    {
+      "epoch": 77.46478873239437,
+      "eval_accuracy": 0.7556376776922981,
+      "eval_loss": 1.293182611465454,
+      "eval_runtime": 207.9948,
+      "eval_samples_per_second": 85.012,
+      "eval_steps_per_second": 2.659,
+      "step": 33000
+    },
+    {
+      "epoch": 78.63849765258216,
+      "grad_norm": 2.865656614303589,
+      "learning_rate": 6.65e-05,
+      "loss": 0.8323,
+      "step": 33500
+    },
+    {
+      "epoch": 78.63849765258216,
+      "eval_accuracy": 0.7562831794705681,
+      "eval_loss": 1.2910946607589722,
+      "eval_runtime": 206.5563,
+      "eval_samples_per_second": 85.604,
+      "eval_steps_per_second": 2.677,
+      "step": 33500
+    },
+    {
+      "epoch": 79.81220657276995,
+      "grad_norm": 3.0894510746002197,
+      "learning_rate": 6.6e-05,
+      "loss": 0.8212,
+      "step": 34000
+    },
+    {
+      "epoch": 79.81220657276995,
+      "eval_accuracy": 0.7566236691052538,
+      "eval_loss": 1.276480793952942,
+      "eval_runtime": 214.1903,
+      "eval_samples_per_second": 82.553,
+      "eval_steps_per_second": 2.582,
+      "step": 34000
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 100000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 235,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.8710040568961434e+17,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86e3757526f7d7016f7eba5993e1a06fbe37cbbd636a32c94d064a5e1bc8f3b6
+size 5176