🚀 Full upload with token redacted

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+mt5_training_data-1.jsonl filter=lfs diff=lfs merge=lfs -text

mt5-finetuned/checkpoint-3396/config.json

@@ -0,0 +1,31 @@
+{
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "classifier_dropout": 0.0,
+  "d_ff": 1024,
+  "d_kv": 64,
+  "d_model": 512,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "mt5",
+  "num_decoder_layers": 8,
+  "num_heads": 6,
+  "num_layers": 8,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.50.3",
+  "use_cache": true,
+  "vocab_size": 250112
+}

mt5-finetuned/checkpoint-3396/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.50.3"
+}

mt5-finetuned/checkpoint-3396/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12c79e43eb66e66bcb13da69d4451680d8712e6b0628c70711a82af7e6c57ed8
+size 1200729512

mt5-finetuned/checkpoint-3396/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6101cc68079cd153e117fe790fe37419ddba2b8d0e219425e1f58144677408cd
+size 2401574330

mt5-finetuned/checkpoint-3396/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbc85351130da129c85f266e528de4e2ebc79b887b0384adec3bb21d7a490d09
+size 14244

mt5-finetuned/checkpoint-3396/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1022e6be58b62a94302bc1721732cafa60a463bf5853b07dee00ba361293d57
+size 1064

mt5-finetuned/checkpoint-3396/trainer_state.json

@@ -0,0 +1,536 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 100,
+  "global_step": 3396,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0005889281507656066,
+      "grad_norm": 1069.6029052734375,
+      "learning_rate": 1.6339869281045752e-07,
+      "loss": 61.3399,
+      "step": 1
+    },
+    {
+      "epoch": 0.05889281507656066,
+      "grad_norm": 295.9472961425781,
+      "learning_rate": 1.6339869281045753e-05,
+      "loss": 55.7859,
+      "step": 100
+    },
+    {
+      "epoch": 0.05889281507656066,
+      "eval_loss": 34.09705352783203,
+      "eval_runtime": 0.5351,
+      "eval_samples_per_second": 934.481,
+      "eval_steps_per_second": 3.738,
+      "step": 100
+    },
+    {
+      "epoch": 0.11778563015312132,
+      "grad_norm": 397.3686218261719,
+      "learning_rate": 3.2679738562091506e-05,
+      "loss": 36.4484,
+      "step": 200
+    },
+    {
+      "epoch": 0.11778563015312132,
+      "eval_loss": 18.607961654663086,
+      "eval_runtime": 0.5345,
+      "eval_samples_per_second": 935.528,
+      "eval_steps_per_second": 3.742,
+      "step": 200
+    },
+    {
+      "epoch": 0.17667844522968199,
+      "grad_norm": 91.78496551513672,
+      "learning_rate": 4.901960784313725e-05,
+      "loss": 19.2835,
+      "step": 300
+    },
+    {
+      "epoch": 0.17667844522968199,
+      "eval_loss": 8.0890474319458,
+      "eval_runtime": 0.542,
+      "eval_samples_per_second": 922.533,
+      "eval_steps_per_second": 3.69,
+      "step": 300
+    },
+    {
+      "epoch": 0.23557126030624265,
+      "grad_norm": 25.501201629638672,
+      "learning_rate": 4.901837928153718e-05,
+      "loss": 7.4464,
+      "step": 400
+    },
+    {
+      "epoch": 0.23557126030624265,
+      "eval_loss": 3.304853677749634,
+      "eval_runtime": 0.5365,
+      "eval_samples_per_second": 931.913,
+      "eval_steps_per_second": 3.728,
+      "step": 400
+    },
+    {
+      "epoch": 0.2944640753828033,
+      "grad_norm": 23.07789421081543,
+      "learning_rate": 4.7974101921470346e-05,
+      "loss": 2.2278,
+      "step": 500
+    },
+    {
+      "epoch": 0.2944640753828033,
+      "eval_loss": 0.6309428811073303,
+      "eval_runtime": 0.534,
+      "eval_samples_per_second": 936.293,
+      "eval_steps_per_second": 3.745,
+      "step": 500
+    },
+    {
+      "epoch": 0.35335689045936397,
+      "grad_norm": 6.330116271972656,
+      "learning_rate": 4.6929824561403515e-05,
+      "loss": 0.7991,
+      "step": 600
+    },
+    {
+      "epoch": 0.35335689045936397,
+      "eval_loss": 0.39563897252082825,
+      "eval_runtime": 0.5351,
+      "eval_samples_per_second": 934.394,
+      "eval_steps_per_second": 3.738,
+      "step": 600
+    },
+    {
+      "epoch": 0.4122497055359246,
+      "grad_norm": 0.8220232129096985,
+      "learning_rate": 4.588554720133668e-05,
+      "loss": 0.4644,
+      "step": 700
+    },
+    {
+      "epoch": 0.4122497055359246,
+      "eval_loss": 0.22928042709827423,
+      "eval_runtime": 0.545,
+      "eval_samples_per_second": 917.424,
+      "eval_steps_per_second": 3.67,
+      "step": 700
+    },
+    {
+      "epoch": 0.4711425206124853,
+      "grad_norm": 0.47425171732902527,
+      "learning_rate": 4.4841269841269846e-05,
+      "loss": 0.3034,
+      "step": 800
+    },
+    {
+      "epoch": 0.4711425206124853,
+      "eval_loss": 0.15583929419517517,
+      "eval_runtime": 0.5482,
+      "eval_samples_per_second": 912.069,
+      "eval_steps_per_second": 3.648,
+      "step": 800
+    },
+    {
+      "epoch": 0.5300353356890459,
+      "grad_norm": 0.33976656198501587,
+      "learning_rate": 4.379699248120301e-05,
+      "loss": 0.2374,
+      "step": 900
+    },
+    {
+      "epoch": 0.5300353356890459,
+      "eval_loss": 0.13674509525299072,
+      "eval_runtime": 0.5433,
+      "eval_samples_per_second": 920.268,
+      "eval_steps_per_second": 3.681,
+      "step": 900
+    },
+    {
+      "epoch": 0.5889281507656066,
+      "grad_norm": 0.29963016510009766,
+      "learning_rate": 4.2752715121136177e-05,
+      "loss": 0.2106,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5889281507656066,
+      "eval_loss": 0.12364959716796875,
+      "eval_runtime": 0.5453,
+      "eval_samples_per_second": 916.855,
+      "eval_steps_per_second": 3.667,
+      "step": 1000
+    },
+    {
+      "epoch": 0.6478209658421673,
+      "grad_norm": 0.21829353272914886,
+      "learning_rate": 4.170843776106934e-05,
+      "loss": 0.1869,
+      "step": 1100
+    },
+    {
+      "epoch": 0.6478209658421673,
+      "eval_loss": 0.11144684255123138,
+      "eval_runtime": 0.5369,
+      "eval_samples_per_second": 931.332,
+      "eval_steps_per_second": 3.725,
+      "step": 1100
+    },
+    {
+      "epoch": 0.7067137809187279,
+      "grad_norm": 0.3481610119342804,
+      "learning_rate": 4.066416040100251e-05,
+      "loss": 0.1718,
+      "step": 1200
+    },
+    {
+      "epoch": 0.7067137809187279,
+      "eval_loss": 0.10174528509378433,
+      "eval_runtime": 0.5358,
+      "eval_samples_per_second": 933.262,
+      "eval_steps_per_second": 3.733,
+      "step": 1200
+    },
+    {
+      "epoch": 0.7656065959952886,
+      "grad_norm": 0.20769686996936798,
+      "learning_rate": 3.9619883040935676e-05,
+      "loss": 0.1608,
+      "step": 1300
+    },
+    {
+      "epoch": 0.7656065959952886,
+      "eval_loss": 0.09667050093412399,
+      "eval_runtime": 0.5354,
+      "eval_samples_per_second": 933.831,
+      "eval_steps_per_second": 3.735,
+      "step": 1300
+    },
+    {
+      "epoch": 0.8244994110718492,
+      "grad_norm": 0.18638956546783447,
+      "learning_rate": 3.8575605680868845e-05,
+      "loss": 0.1527,
+      "step": 1400
+    },
+    {
+      "epoch": 0.8244994110718492,
+      "eval_loss": 0.09282659739255905,
+      "eval_runtime": 0.5367,
+      "eval_samples_per_second": 931.64,
+      "eval_steps_per_second": 3.727,
+      "step": 1400
+    },
+    {
+      "epoch": 0.8833922261484098,
+      "grad_norm": 0.1558378040790558,
+      "learning_rate": 3.753132832080201e-05,
+      "loss": 0.1483,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8833922261484098,
+      "eval_loss": 0.0904233381152153,
+      "eval_runtime": 0.5394,
+      "eval_samples_per_second": 926.881,
+      "eval_steps_per_second": 3.708,
+      "step": 1500
+    },
+    {
+      "epoch": 0.9422850412249706,
+      "grad_norm": 0.15275876224040985,
+      "learning_rate": 3.6487050960735175e-05,
+      "loss": 0.1412,
+      "step": 1600
+    },
+    {
+      "epoch": 0.9422850412249706,
+      "eval_loss": 0.08748902380466461,
+      "eval_runtime": 0.5405,
+      "eval_samples_per_second": 925.022,
+      "eval_steps_per_second": 3.7,
+      "step": 1600
+    },
+    {
+      "epoch": 1.0011778563015312,
+      "grad_norm": 0.20647253096103668,
+      "learning_rate": 3.544277360066834e-05,
+      "loss": 0.1372,
+      "step": 1700
+    },
+    {
+      "epoch": 1.0011778563015312,
+      "eval_loss": 0.08554470539093018,
+      "eval_runtime": 0.5347,
+      "eval_samples_per_second": 935.021,
+      "eval_steps_per_second": 3.74,
+      "step": 1700
+    },
+    {
+      "epoch": 1.0600706713780919,
+      "grad_norm": 0.13815245032310486,
+      "learning_rate": 3.4398496240601506e-05,
+      "loss": 0.1325,
+      "step": 1800
+    },
+    {
+      "epoch": 1.0600706713780919,
+      "eval_loss": 0.0836854949593544,
+      "eval_runtime": 0.5352,
+      "eval_samples_per_second": 934.268,
+      "eval_steps_per_second": 3.737,
+      "step": 1800
+    },
+    {
+      "epoch": 1.1189634864546525,
+      "grad_norm": 0.14026539027690887,
+      "learning_rate": 3.335421888053467e-05,
+      "loss": 0.13,
+      "step": 1900
+    },
+    {
+      "epoch": 1.1189634864546525,
+      "eval_loss": 0.08203620463609695,
+      "eval_runtime": 0.5348,
+      "eval_samples_per_second": 934.996,
+      "eval_steps_per_second": 3.74,
+      "step": 1900
+    },
+    {
+      "epoch": 1.1778563015312131,
+      "grad_norm": 0.12261384725570679,
+      "learning_rate": 3.230994152046784e-05,
+      "loss": 0.1282,
+      "step": 2000
+    },
+    {
+      "epoch": 1.1778563015312131,
+      "eval_loss": 0.08088693022727966,
+      "eval_runtime": 0.539,
+      "eval_samples_per_second": 927.574,
+      "eval_steps_per_second": 3.71,
+      "step": 2000
+    },
+    {
+      "epoch": 1.2367491166077738,
+      "grad_norm": 0.13724654912948608,
+      "learning_rate": 3.1265664160401006e-05,
+      "loss": 0.1247,
+      "step": 2100
+    },
+    {
+      "epoch": 1.2367491166077738,
+      "eval_loss": 0.07952813804149628,
+      "eval_runtime": 0.5404,
+      "eval_samples_per_second": 925.283,
+      "eval_steps_per_second": 3.701,
+      "step": 2100
+    },
+    {
+      "epoch": 1.2956419316843344,
+      "grad_norm": 0.17809857428073883,
+      "learning_rate": 3.022138680033417e-05,
+      "loss": 0.1238,
+      "step": 2200
+    },
+    {
+      "epoch": 1.2956419316843344,
+      "eval_loss": 0.07879804819822311,
+      "eval_runtime": 0.5341,
+      "eval_samples_per_second": 936.116,
+      "eval_steps_per_second": 3.744,
+      "step": 2200
+    },
+    {
+      "epoch": 1.3545347467608952,
+      "grad_norm": 0.11621029675006866,
+      "learning_rate": 2.9177109440267336e-05,
+      "loss": 0.1209,
+      "step": 2300
+    },
+    {
+      "epoch": 1.3545347467608952,
+      "eval_loss": 0.07757514715194702,
+      "eval_runtime": 0.5351,
+      "eval_samples_per_second": 934.37,
+      "eval_steps_per_second": 3.737,
+      "step": 2300
+    },
+    {
+      "epoch": 1.4134275618374559,
+      "grad_norm": 0.12245041131973267,
+      "learning_rate": 2.8132832080200505e-05,
+      "loss": 0.1166,
+      "step": 2400
+    },
+    {
+      "epoch": 1.4134275618374559,
+      "eval_loss": 0.07674014568328857,
+      "eval_runtime": 0.5335,
+      "eval_samples_per_second": 937.176,
+      "eval_steps_per_second": 3.749,
+      "step": 2400
+    },
+    {
+      "epoch": 1.4723203769140165,
+      "grad_norm": 0.11454136669635773,
+      "learning_rate": 2.7088554720133667e-05,
+      "loss": 0.1189,
+      "step": 2500
+    },
+    {
+      "epoch": 1.4723203769140165,
+      "eval_loss": 0.0758647620677948,
+      "eval_runtime": 0.5451,
+      "eval_samples_per_second": 917.286,
+      "eval_steps_per_second": 3.669,
+      "step": 2500
+    },
+    {
+      "epoch": 1.5312131919905771,
+      "grad_norm": 0.10498815774917603,
+      "learning_rate": 2.604427736006684e-05,
+      "loss": 0.1154,
+      "step": 2600
+    },
+    {
+      "epoch": 1.5312131919905771,
+      "eval_loss": 0.07501054555177689,
+      "eval_runtime": 0.5354,
+      "eval_samples_per_second": 933.938,
+      "eval_steps_per_second": 3.736,
+      "step": 2600
+    },
+    {
+      "epoch": 1.5901060070671378,
+      "grad_norm": 0.1041310578584671,
+      "learning_rate": 2.5e-05,
+      "loss": 0.1145,
+      "step": 2700
+    },
+    {
+      "epoch": 1.5901060070671378,
+      "eval_loss": 0.07420270144939423,
+      "eval_runtime": 0.5438,
+      "eval_samples_per_second": 919.476,
+      "eval_steps_per_second": 3.678,
+      "step": 2700
+    },
+    {
+      "epoch": 1.6489988221436984,
+      "grad_norm": 0.14854931831359863,
+      "learning_rate": 2.3955722639933167e-05,
+      "loss": 0.1123,
+      "step": 2800
+    },
+    {
+      "epoch": 1.6489988221436984,
+      "eval_loss": 0.07348344475030899,
+      "eval_runtime": 0.5481,
+      "eval_samples_per_second": 912.284,
+      "eval_steps_per_second": 3.649,
+      "step": 2800
+    },
+    {
+      "epoch": 1.7078916372202593,
+      "grad_norm": 0.10635272413492203,
+      "learning_rate": 2.2911445279866335e-05,
+      "loss": 0.1127,
+      "step": 2900
+    },
+    {
+      "epoch": 1.7078916372202593,
+      "eval_loss": 0.07267069816589355,
+      "eval_runtime": 0.5354,
+      "eval_samples_per_second": 933.878,
+      "eval_steps_per_second": 3.736,
+      "step": 2900
+    },
+    {
+      "epoch": 1.76678445229682,
+      "grad_norm": 0.11862709373235703,
+      "learning_rate": 2.18671679197995e-05,
+      "loss": 0.1109,
+      "step": 3000
+    },
+    {
+      "epoch": 1.76678445229682,
+      "eval_loss": 0.07196911424398422,
+      "eval_runtime": 0.5364,
+      "eval_samples_per_second": 932.187,
+      "eval_steps_per_second": 3.729,
+      "step": 3000
+    },
+    {
+      "epoch": 1.8256772673733805,
+      "grad_norm": 0.11262491345405579,
+      "learning_rate": 2.0822890559732666e-05,
+      "loss": 0.11,
+      "step": 3100
+    },
+    {
+      "epoch": 1.8256772673733805,
+      "eval_loss": 0.0713561549782753,
+      "eval_runtime": 0.5407,
+      "eval_samples_per_second": 924.79,
+      "eval_steps_per_second": 3.699,
+      "step": 3100
+    },
+    {
+      "epoch": 1.8845700824499412,
+      "grad_norm": 0.17745310068130493,
+      "learning_rate": 1.977861319966583e-05,
+      "loss": 0.1084,
+      "step": 3200
+    },
+    {
+      "epoch": 1.8845700824499412,
+      "eval_loss": 0.07110374420881271,
+      "eval_runtime": 0.537,
+      "eval_samples_per_second": 931.079,
+      "eval_steps_per_second": 3.724,
+      "step": 3200
+    },
+    {
+      "epoch": 1.9434628975265018,
+      "grad_norm": 0.12771931290626526,
+      "learning_rate": 1.8734335839599e-05,
+      "loss": 0.1083,
+      "step": 3300
+    },
+    {
+      "epoch": 1.9434628975265018,
+      "eval_loss": 0.07047487050294876,
+      "eval_runtime": 0.5416,
+      "eval_samples_per_second": 923.161,
+      "eval_steps_per_second": 3.693,
+      "step": 3300
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 5094,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.2439819631132672e+17,
+  "train_batch_size": 250,
+  "trial_name": null,
+  "trial_params": null
+}

mt5-finetuned/checkpoint-3396/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:338671f41322537cc4ced53c635facb13cc93423bbd52c98d16f5afbe30f2376
+size 5304

mt5-finetuned/checkpoint-5094/config.json

@@ -0,0 +1,31 @@
+{
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "classifier_dropout": 0.0,
+  "d_ff": 1024,
+  "d_kv": 64,
+  "d_model": 512,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "mt5",
+  "num_decoder_layers": 8,
+  "num_heads": 6,
+  "num_layers": 8,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.50.3",
+  "use_cache": true,
+  "vocab_size": 250112
+}

mt5-finetuned/checkpoint-5094/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.50.3"
+}

mt5-finetuned/checkpoint-5094/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0529a72bcfa99e852266c278f9fe2be272ea49629b4ec8734831136e6bc4645
+size 1200729512

mt5-finetuned/checkpoint-5094/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c5b8bdc4d47ec94aa1c9b9cb5e3c924ce4111bd634796a8aed15fcf87899535
+size 2401574330

mt5-finetuned/checkpoint-5094/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2c5cb50e12211e8ff6f13dcf72c4e0877e4918fa1a1ef677fddd3262f3553d4
+size 14244

mt5-finetuned/checkpoint-5094/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a510cef9a5cd55b582c9172902c5ea4d5b7dba6acfef03beb1323ced783e71a
+size 1064

mt5-finetuned/checkpoint-5094/trainer_state.json

@@ -0,0 +1,791 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 100,
+  "global_step": 5094,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0005889281507656066,
+      "grad_norm": 1069.6029052734375,
+      "learning_rate": 1.6339869281045752e-07,
+      "loss": 61.3399,
+      "step": 1
+    },
+    {
+      "epoch": 0.05889281507656066,
+      "grad_norm": 295.9472961425781,
+      "learning_rate": 1.6339869281045753e-05,
+      "loss": 55.7859,
+      "step": 100
+    },
+    {
+      "epoch": 0.05889281507656066,
+      "eval_loss": 34.09705352783203,
+      "eval_runtime": 0.5351,
+      "eval_samples_per_second": 934.481,
+      "eval_steps_per_second": 3.738,
+      "step": 100
+    },
+    {
+      "epoch": 0.11778563015312132,
+      "grad_norm": 397.3686218261719,
+      "learning_rate": 3.2679738562091506e-05,
+      "loss": 36.4484,
+      "step": 200
+    },
+    {
+      "epoch": 0.11778563015312132,
+      "eval_loss": 18.607961654663086,
+      "eval_runtime": 0.5345,
+      "eval_samples_per_second": 935.528,
+      "eval_steps_per_second": 3.742,
+      "step": 200
+    },
+    {
+      "epoch": 0.17667844522968199,
+      "grad_norm": 91.78496551513672,
+      "learning_rate": 4.901960784313725e-05,
+      "loss": 19.2835,
+      "step": 300
+    },
+    {
+      "epoch": 0.17667844522968199,
+      "eval_loss": 8.0890474319458,
+      "eval_runtime": 0.542,
+      "eval_samples_per_second": 922.533,
+      "eval_steps_per_second": 3.69,
+      "step": 300
+    },
+    {
+      "epoch": 0.23557126030624265,
+      "grad_norm": 25.501201629638672,
+      "learning_rate": 4.901837928153718e-05,
+      "loss": 7.4464,
+      "step": 400
+    },
+    {
+      "epoch": 0.23557126030624265,
+      "eval_loss": 3.304853677749634,
+      "eval_runtime": 0.5365,
+      "eval_samples_per_second": 931.913,
+      "eval_steps_per_second": 3.728,
+      "step": 400
+    },
+    {
+      "epoch": 0.2944640753828033,
+      "grad_norm": 23.07789421081543,
+      "learning_rate": 4.7974101921470346e-05,
+      "loss": 2.2278,
+      "step": 500
+    },
+    {
+      "epoch": 0.2944640753828033,
+      "eval_loss": 0.6309428811073303,
+      "eval_runtime": 0.534,
+      "eval_samples_per_second": 936.293,
+      "eval_steps_per_second": 3.745,
+      "step": 500
+    },
+    {
+      "epoch": 0.35335689045936397,
+      "grad_norm": 6.330116271972656,
+      "learning_rate": 4.6929824561403515e-05,
+      "loss": 0.7991,
+      "step": 600
+    },
+    {
+      "epoch": 0.35335689045936397,
+      "eval_loss": 0.39563897252082825,
+      "eval_runtime": 0.5351,
+      "eval_samples_per_second": 934.394,
+      "eval_steps_per_second": 3.738,
+      "step": 600
+    },
+    {
+      "epoch": 0.4122497055359246,
+      "grad_norm": 0.8220232129096985,
+      "learning_rate": 4.588554720133668e-05,
+      "loss": 0.4644,
+      "step": 700
+    },
+    {
+      "epoch": 0.4122497055359246,
+      "eval_loss": 0.22928042709827423,
+      "eval_runtime": 0.545,
+      "eval_samples_per_second": 917.424,
+      "eval_steps_per_second": 3.67,
+      "step": 700
+    },
+    {
+      "epoch": 0.4711425206124853,
+      "grad_norm": 0.47425171732902527,
+      "learning_rate": 4.4841269841269846e-05,
+      "loss": 0.3034,
+      "step": 800
+    },
+    {
+      "epoch": 0.4711425206124853,
+      "eval_loss": 0.15583929419517517,
+      "eval_runtime": 0.5482,
+      "eval_samples_per_second": 912.069,
+      "eval_steps_per_second": 3.648,
+      "step": 800
+    },
+    {
+      "epoch": 0.5300353356890459,
+      "grad_norm": 0.33976656198501587,
+      "learning_rate": 4.379699248120301e-05,
+      "loss": 0.2374,
+      "step": 900
+    },
+    {
+      "epoch": 0.5300353356890459,
+      "eval_loss": 0.13674509525299072,
+      "eval_runtime": 0.5433,
+      "eval_samples_per_second": 920.268,
+      "eval_steps_per_second": 3.681,
+      "step": 900
+    },
+    {
+      "epoch": 0.5889281507656066,
+      "grad_norm": 0.29963016510009766,
+      "learning_rate": 4.2752715121136177e-05,
+      "loss": 0.2106,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5889281507656066,
+      "eval_loss": 0.12364959716796875,
+      "eval_runtime": 0.5453,
+      "eval_samples_per_second": 916.855,
+      "eval_steps_per_second": 3.667,
+      "step": 1000
+    },
+    {
+      "epoch": 0.6478209658421673,
+      "grad_norm": 0.21829353272914886,
+      "learning_rate": 4.170843776106934e-05,
+      "loss": 0.1869,
+      "step": 1100
+    },
+    {
+      "epoch": 0.6478209658421673,
+      "eval_loss": 0.11144684255123138,
+      "eval_runtime": 0.5369,
+      "eval_samples_per_second": 931.332,
+      "eval_steps_per_second": 3.725,
+      "step": 1100
+    },
+    {
+      "epoch": 0.7067137809187279,
+      "grad_norm": 0.3481610119342804,
+      "learning_rate": 4.066416040100251e-05,
+      "loss": 0.1718,
+      "step": 1200
+    },
+    {
+      "epoch": 0.7067137809187279,
+      "eval_loss": 0.10174528509378433,
+      "eval_runtime": 0.5358,
+      "eval_samples_per_second": 933.262,
+      "eval_steps_per_second": 3.733,
+      "step": 1200
+    },
+    {
+      "epoch": 0.7656065959952886,
+      "grad_norm": 0.20769686996936798,
+      "learning_rate": 3.9619883040935676e-05,
+      "loss": 0.1608,
+      "step": 1300
+    },
+    {
+      "epoch": 0.7656065959952886,
+      "eval_loss": 0.09667050093412399,
+      "eval_runtime": 0.5354,
+      "eval_samples_per_second": 933.831,
+      "eval_steps_per_second": 3.735,
+      "step": 1300
+    },
+    {
+      "epoch": 0.8244994110718492,
+      "grad_norm": 0.18638956546783447,
+      "learning_rate": 3.8575605680868845e-05,
+      "loss": 0.1527,
+      "step": 1400
+    },
+    {
+      "epoch": 0.8244994110718492,
+      "eval_loss": 0.09282659739255905,
+      "eval_runtime": 0.5367,
+      "eval_samples_per_second": 931.64,
+      "eval_steps_per_second": 3.727,
+      "step": 1400
+    },
+    {
+      "epoch": 0.8833922261484098,
+      "grad_norm": 0.1558378040790558,
+      "learning_rate": 3.753132832080201e-05,
+      "loss": 0.1483,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8833922261484098,
+      "eval_loss": 0.0904233381152153,
+      "eval_runtime": 0.5394,
+      "eval_samples_per_second": 926.881,
+      "eval_steps_per_second": 3.708,
+      "step": 1500
+    },
+    {
+      "epoch": 0.9422850412249706,
+      "grad_norm": 0.15275876224040985,
+      "learning_rate": 3.6487050960735175e-05,
+      "loss": 0.1412,
+      "step": 1600
+    },
+    {
+      "epoch": 0.9422850412249706,
+      "eval_loss": 0.08748902380466461,
+      "eval_runtime": 0.5405,
+      "eval_samples_per_second": 925.022,
+      "eval_steps_per_second": 3.7,
+      "step": 1600
+    },
+    {
+      "epoch": 1.0011778563015312,
+      "grad_norm": 0.20647253096103668,
+      "learning_rate": 3.544277360066834e-05,
+      "loss": 0.1372,
+      "step": 1700
+    },
+    {
+      "epoch": 1.0011778563015312,
+      "eval_loss": 0.08554470539093018,
+      "eval_runtime": 0.5347,
+      "eval_samples_per_second": 935.021,
+      "eval_steps_per_second": 3.74,
+      "step": 1700
+    },
+    {
+      "epoch": 1.0600706713780919,
+      "grad_norm": 0.13815245032310486,
+      "learning_rate": 3.4398496240601506e-05,
+      "loss": 0.1325,
+      "step": 1800
+    },
+    {
+      "epoch": 1.0600706713780919,
+      "eval_loss": 0.0836854949593544,
+      "eval_runtime": 0.5352,
+      "eval_samples_per_second": 934.268,
+      "eval_steps_per_second": 3.737,
+      "step": 1800
+    },
+    {
+      "epoch": 1.1189634864546525,
+      "grad_norm": 0.14026539027690887,
+      "learning_rate": 3.335421888053467e-05,
+      "loss": 0.13,
+      "step": 1900
+    },
+    {
+      "epoch": 1.1189634864546525,
+      "eval_loss": 0.08203620463609695,
+      "eval_runtime": 0.5348,
+      "eval_samples_per_second": 934.996,
+      "eval_steps_per_second": 3.74,
+      "step": 1900
+    },
+    {
+      "epoch": 1.1778563015312131,
+      "grad_norm": 0.12261384725570679,
+      "learning_rate": 3.230994152046784e-05,
+      "loss": 0.1282,
+      "step": 2000
+    },
+    {
+      "epoch": 1.1778563015312131,
+      "eval_loss": 0.08088693022727966,
+      "eval_runtime": 0.539,
+      "eval_samples_per_second": 927.574,
+      "eval_steps_per_second": 3.71,
+      "step": 2000
+    },
+    {
+      "epoch": 1.2367491166077738,
+      "grad_norm": 0.13724654912948608,
+      "learning_rate": 3.1265664160401006e-05,
+      "loss": 0.1247,
+      "step": 2100
+    },
+    {
+      "epoch": 1.2367491166077738,
+      "eval_loss": 0.07952813804149628,
+      "eval_runtime": 0.5404,
+      "eval_samples_per_second": 925.283,
+      "eval_steps_per_second": 3.701,
+      "step": 2100
+    },
+    {
+      "epoch": 1.2956419316843344,
+      "grad_norm": 0.17809857428073883,
+      "learning_rate": 3.022138680033417e-05,
+      "loss": 0.1238,
+      "step": 2200
+    },
+    {
+      "epoch": 1.2956419316843344,
+      "eval_loss": 0.07879804819822311,
+      "eval_runtime": 0.5341,
+      "eval_samples_per_second": 936.116,
+      "eval_steps_per_second": 3.744,
+      "step": 2200
+    },
+    {
+      "epoch": 1.3545347467608952,
+      "grad_norm": 0.11621029675006866,
+      "learning_rate": 2.9177109440267336e-05,
+      "loss": 0.1209,
+      "step": 2300
+    },
+    {
+      "epoch": 1.3545347467608952,
+      "eval_loss": 0.07757514715194702,
+      "eval_runtime": 0.5351,
+      "eval_samples_per_second": 934.37,
+      "eval_steps_per_second": 3.737,
+      "step": 2300
+    },
+    {
+      "epoch": 1.4134275618374559,
+      "grad_norm": 0.12245041131973267,
+      "learning_rate": 2.8132832080200505e-05,
+      "loss": 0.1166,
+      "step": 2400
+    },
+    {
+      "epoch": 1.4134275618374559,
+      "eval_loss": 0.07674014568328857,
+      "eval_runtime": 0.5335,
+      "eval_samples_per_second": 937.176,
+      "eval_steps_per_second": 3.749,
+      "step": 2400
+    },
+    {
+      "epoch": 1.4723203769140165,
+      "grad_norm": 0.11454136669635773,
+      "learning_rate": 2.7088554720133667e-05,
+      "loss": 0.1189,
+      "step": 2500
+    },
+    {
+      "epoch": 1.4723203769140165,
+      "eval_loss": 0.0758647620677948,
+      "eval_runtime": 0.5451,
+      "eval_samples_per_second": 917.286,
+      "eval_steps_per_second": 3.669,
+      "step": 2500
+    },
+    {
+      "epoch": 1.5312131919905771,
+      "grad_norm": 0.10498815774917603,
+      "learning_rate": 2.604427736006684e-05,
+      "loss": 0.1154,
+      "step": 2600
+    },
+    {
+      "epoch": 1.5312131919905771,
+      "eval_loss": 0.07501054555177689,
+      "eval_runtime": 0.5354,
+      "eval_samples_per_second": 933.938,
+      "eval_steps_per_second": 3.736,
+      "step": 2600
+    },
+    {
+      "epoch": 1.5901060070671378,
+      "grad_norm": 0.1041310578584671,
+      "learning_rate": 2.5e-05,
+      "loss": 0.1145,
+      "step": 2700
+    },
+    {
+      "epoch": 1.5901060070671378,
+      "eval_loss": 0.07420270144939423,
+      "eval_runtime": 0.5438,
+      "eval_samples_per_second": 919.476,
+      "eval_steps_per_second": 3.678,
+      "step": 2700
+    },
+    {
+      "epoch": 1.6489988221436984,
+      "grad_norm": 0.14854931831359863,
+      "learning_rate": 2.3955722639933167e-05,
+      "loss": 0.1123,
+      "step": 2800
+    },
+    {
+      "epoch": 1.6489988221436984,
+      "eval_loss": 0.07348344475030899,
+      "eval_runtime": 0.5481,
+      "eval_samples_per_second": 912.284,
+      "eval_steps_per_second": 3.649,
+      "step": 2800
+    },
+    {
+      "epoch": 1.7078916372202593,
+      "grad_norm": 0.10635272413492203,
+      "learning_rate": 2.2911445279866335e-05,
+      "loss": 0.1127,
+      "step": 2900
+    },
+    {
+      "epoch": 1.7078916372202593,
+      "eval_loss": 0.07267069816589355,
+      "eval_runtime": 0.5354,
+      "eval_samples_per_second": 933.878,
+      "eval_steps_per_second": 3.736,
+      "step": 2900
+    },
+    {
+      "epoch": 1.76678445229682,
+      "grad_norm": 0.11862709373235703,
+      "learning_rate": 2.18671679197995e-05,
+      "loss": 0.1109,
+      "step": 3000
+    },
+    {
+      "epoch": 1.76678445229682,
+      "eval_loss": 0.07196911424398422,
+      "eval_runtime": 0.5364,
+      "eval_samples_per_second": 932.187,
+      "eval_steps_per_second": 3.729,
+      "step": 3000
+    },
+    {
+      "epoch": 1.8256772673733805,
+      "grad_norm": 0.11262491345405579,
+      "learning_rate": 2.0822890559732666e-05,
+      "loss": 0.11,
+      "step": 3100
+    },
+    {
+      "epoch": 1.8256772673733805,
+      "eval_loss": 0.0713561549782753,
+      "eval_runtime": 0.5407,
+      "eval_samples_per_second": 924.79,
+      "eval_steps_per_second": 3.699,
+      "step": 3100
+    },
+    {
+      "epoch": 1.8845700824499412,
+      "grad_norm": 0.17745310068130493,
+      "learning_rate": 1.977861319966583e-05,
+      "loss": 0.1084,
+      "step": 3200
+    },
+    {
+      "epoch": 1.8845700824499412,
+      "eval_loss": 0.07110374420881271,
+      "eval_runtime": 0.537,
+      "eval_samples_per_second": 931.079,
+      "eval_steps_per_second": 3.724,
+      "step": 3200
+    },
+    {
+      "epoch": 1.9434628975265018,
+      "grad_norm": 0.12771931290626526,
+      "learning_rate": 1.8734335839599e-05,
+      "loss": 0.1083,
+      "step": 3300
+    },
+    {
+      "epoch": 1.9434628975265018,
+      "eval_loss": 0.07047487050294876,
+      "eval_runtime": 0.5416,
+      "eval_samples_per_second": 923.161,
+      "eval_steps_per_second": 3.693,
+      "step": 3300
+    },
+    {
+      "epoch": 2.0023557126030624,
+      "grad_norm": 0.10948721319437027,
+      "learning_rate": 1.7690058479532165e-05,
+      "loss": 0.1087,
+      "step": 3400
+    },
+    {
+      "epoch": 2.0023557126030624,
+      "eval_loss": 0.06981312483549118,
+      "eval_runtime": 0.5441,
+      "eval_samples_per_second": 918.969,
+      "eval_steps_per_second": 3.676,
+      "step": 3400
+    },
+    {
+      "epoch": 2.061248527679623,
+      "grad_norm": 0.10693700611591339,
+      "learning_rate": 1.664578111946533e-05,
+      "loss": 0.1058,
+      "step": 3500
+    },
+    {
+      "epoch": 2.061248527679623,
+      "eval_loss": 0.06955926865339279,
+      "eval_runtime": 0.5359,
+      "eval_samples_per_second": 933.089,
+      "eval_steps_per_second": 3.732,
+      "step": 3500
+    },
+    {
+      "epoch": 2.1201413427561837,
+      "grad_norm": 0.13173066079616547,
+      "learning_rate": 1.5601503759398496e-05,
+      "loss": 0.1053,
+      "step": 3600
+    },
+    {
+      "epoch": 2.1201413427561837,
+      "eval_loss": 0.06913256645202637,
+      "eval_runtime": 0.5364,
+      "eval_samples_per_second": 932.186,
+      "eval_steps_per_second": 3.729,
+      "step": 3600
+    },
+    {
+      "epoch": 2.1790341578327443,
+      "grad_norm": 0.09946483373641968,
+      "learning_rate": 1.4557226399331663e-05,
+      "loss": 0.1051,
+      "step": 3700
+    },
+    {
+      "epoch": 2.1790341578327443,
+      "eval_loss": 0.06875628232955933,
+      "eval_runtime": 0.5347,
+      "eval_samples_per_second": 935.082,
+      "eval_steps_per_second": 3.74,
+      "step": 3700
+    },
+    {
+      "epoch": 2.237926972909305,
+      "grad_norm": 0.12029112130403519,
+      "learning_rate": 1.351294903926483e-05,
+      "loss": 0.1044,
+      "step": 3800
+    },
+    {
+      "epoch": 2.237926972909305,
+      "eval_loss": 0.06848787516355515,
+      "eval_runtime": 0.539,
+      "eval_samples_per_second": 927.692,
+      "eval_steps_per_second": 3.711,
+      "step": 3800
+    },
+    {
+      "epoch": 2.2968197879858656,
+      "grad_norm": 0.11469805240631104,
+      "learning_rate": 1.2468671679197996e-05,
+      "loss": 0.1027,
+      "step": 3900
+    },
+    {
+      "epoch": 2.2968197879858656,
+      "eval_loss": 0.06818344444036484,
+      "eval_runtime": 0.5338,
+      "eval_samples_per_second": 936.719,
+      "eval_steps_per_second": 3.747,
+      "step": 3900
+    },
+    {
+      "epoch": 2.3557126030624262,
+      "grad_norm": 0.09450303018093109,
+      "learning_rate": 1.1424394319131161e-05,
+      "loss": 0.1038,
+      "step": 4000
+    },
+    {
+      "epoch": 2.3557126030624262,
+      "eval_loss": 0.06780162453651428,
+      "eval_runtime": 0.5352,
+      "eval_samples_per_second": 934.248,
+      "eval_steps_per_second": 3.737,
+      "step": 4000
+    },
+    {
+      "epoch": 2.414605418138987,
+      "grad_norm": 0.09121797233819962,
+      "learning_rate": 1.0380116959064328e-05,
+      "loss": 0.1033,
+      "step": 4100
+    },
+    {
+      "epoch": 2.414605418138987,
+      "eval_loss": 0.06764357537031174,
+      "eval_runtime": 0.537,
+      "eval_samples_per_second": 931.05,
+      "eval_steps_per_second": 3.724,
+      "step": 4100
+    },
+    {
+      "epoch": 2.4734982332155475,
+      "grad_norm": 0.10826277732849121,
+      "learning_rate": 9.335839598997493e-06,
+      "loss": 0.103,
+      "step": 4200
+    },
+    {
+      "epoch": 2.4734982332155475,
+      "eval_loss": 0.06742237508296967,
+      "eval_runtime": 0.5439,
+      "eval_samples_per_second": 919.306,
+      "eval_steps_per_second": 3.677,
+      "step": 4200
+    },
+    {
+      "epoch": 2.5323910482921086,
+      "grad_norm": 0.10901422053575516,
+      "learning_rate": 8.29156223893066e-06,
+      "loss": 0.1008,
+      "step": 4300
+    },
+    {
+      "epoch": 2.5323910482921086,
+      "eval_loss": 0.06723643094301224,
+      "eval_runtime": 0.5356,
+      "eval_samples_per_second": 933.609,
+      "eval_steps_per_second": 3.734,
+      "step": 4300
+    },
+    {
+      "epoch": 2.591283863368669,
+      "grad_norm": 0.09686878323554993,
+      "learning_rate": 7.247284878863826e-06,
+      "loss": 0.1012,
+      "step": 4400
+    },
+    {
+      "epoch": 2.591283863368669,
+      "eval_loss": 0.06717756390571594,
+      "eval_runtime": 0.5349,
+      "eval_samples_per_second": 934.686,
+      "eval_steps_per_second": 3.739,
+      "step": 4400
+    },
+    {
+      "epoch": 2.65017667844523,
+      "grad_norm": 0.1727057844400406,
+      "learning_rate": 6.203007518796992e-06,
+      "loss": 0.1017,
+      "step": 4500
+    },
+    {
+      "epoch": 2.65017667844523,
+      "eval_loss": 0.06697698682546616,
+      "eval_runtime": 0.5491,
+      "eval_samples_per_second": 910.517,
+      "eval_steps_per_second": 3.642,
+      "step": 4500
+    },
+    {
+      "epoch": 2.7090694935217905,
+      "grad_norm": 0.1121884360909462,
+      "learning_rate": 5.158730158730159e-06,
+      "loss": 0.1004,
+      "step": 4600
+    },
+    {
+      "epoch": 2.7090694935217905,
+      "eval_loss": 0.06690937280654907,
+      "eval_runtime": 0.5371,
+      "eval_samples_per_second": 930.912,
+      "eval_steps_per_second": 3.724,
+      "step": 4600
+    },
+    {
+      "epoch": 2.767962308598351,
+      "grad_norm": 0.08828947693109512,
+      "learning_rate": 4.114452798663325e-06,
+      "loss": 0.0998,
+      "step": 4700
+    },
+    {
+      "epoch": 2.767962308598351,
+      "eval_loss": 0.06677506864070892,
+      "eval_runtime": 0.5482,
+      "eval_samples_per_second": 912.01,
+      "eval_steps_per_second": 3.648,
+      "step": 4700
+    },
+    {
+      "epoch": 2.8268551236749118,
+      "grad_norm": 0.09760947525501251,
+      "learning_rate": 3.070175438596491e-06,
+      "loss": 0.0989,
+      "step": 4800
+    },
+    {
+      "epoch": 2.8268551236749118,
+      "eval_loss": 0.0667320117354393,
+      "eval_runtime": 0.5435,
+      "eval_samples_per_second": 920.015,
+      "eval_steps_per_second": 3.68,
+      "step": 4800
+    },
+    {
+      "epoch": 2.8857479387514724,
+      "grad_norm": 0.0929727628827095,
+      "learning_rate": 2.0258980785296573e-06,
+      "loss": 0.1019,
+      "step": 4900
+    },
+    {
+      "epoch": 2.8857479387514724,
+      "eval_loss": 0.06665968149900436,
+      "eval_runtime": 0.536,
+      "eval_samples_per_second": 932.893,
+      "eval_steps_per_second": 3.732,
+      "step": 4900
+    },
+    {
+      "epoch": 2.944640753828033,
+      "grad_norm": 0.12202879041433334,
+      "learning_rate": 9.816207184628237e-07,
+      "loss": 0.098,
+      "step": 5000
+    },
+    {
+      "epoch": 2.944640753828033,
+      "eval_loss": 0.06658908724784851,
+      "eval_runtime": 0.5374,
+      "eval_samples_per_second": 930.358,
+      "eval_steps_per_second": 3.721,
+      "step": 5000
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 5094,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.365972944669901e+17,
+  "train_batch_size": 250,
+  "trial_name": null,
+  "trial_params": null
+}

mt5-finetuned/checkpoint-5094/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:338671f41322537cc4ced53c635facb13cc93423bbd52c98d16f5afbe30f2376
+size 5304

mt5-finetuned/config.json

@@ -0,0 +1,31 @@
+{
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "classifier_dropout": 0.0,
+  "d_ff": 1024,
+  "d_kv": 64,
+  "d_model": 512,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "mt5",
+  "num_decoder_layers": 8,
+  "num_heads": 6,
+  "num_layers": 8,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.50.3",
+  "use_cache": true,
+  "vocab_size": 250112
+}

mt5-finetuned/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.50.3"
+}

mt5-finetuned/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0529a72bcfa99e852266c278f9fe2be272ea49629b4ec8734831136e6bc4645
+size 1200729512

mt5-finetuned/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

mt5-finetuned/spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

mt5-finetuned/tokenizer_config.json

@@ -0,0 +1,40 @@
+{
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "MT5Tokenizer",
+  "unk_token": "<unk>"
+}

mt5_training_data-1.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efac62c55256372f42df4573729c7d4e0dce2c0046d7572b9712fcffc6c1e9aa
+size 96595844

requirements.txt

@@ -0,0 +1,8 @@
+transformers
+sentencepiece
+datasets
+safetensors
+torch
+accelerate
+scipy
+wandb

save.py

@@ -0,0 +1,45 @@
+from huggingface_hub import upload_folder
+import os
+import tempfile
+import shutil
+
+hf_token = ""***HIDDEN_TOKEN***""
+repo_id = "ajkndfjsdfasdf/mt5-small-bigdataset"
+folder_path = "."
+
+ignored = {".git", "__pycache__", ".ipynb_checkpoints", "wandb", "logs", "temp_hf_repo"}
+
+# 🧼 Временная папка
+clean_dir = tempfile.mkdtemp()
+
+# Копируем все файлы, кроме игнорируемых
+for item in os.listdir(folder_path):
+    if item in ignored:
+        continue
+    src = os.path.join(folder_path, item)
+    dst = os.path.join(clean_dir, item)
+
+    if os.path.isdir(src):
+        shutil.copytree(src, dst, dirs_exist_ok=True)
+    elif os.path.isfile(src):
+        # Если файл .py и содержит токен — заменим токен на "***"
+        if src.endswith(".py"):
+            with open(src, "r") as f:
+                content = f.read()
+            # Маскируем токен в коде
+            content = content.replace(hf_token, '"***HIDDEN_TOKEN***"')
+            with open(dst, "w") as f:
+                f.write(content)
+        else:
+            shutil.copy2(src, dst)
+
+# 🚀 Загружаем на Hugging Face
+upload_folder(
+    repo_id=repo_id,
+    folder_path=clean_dir,
+    repo_type="model",
+    token=hf_token,
+    commit_message="🚀 Full upload with token redacted"
+)
+
+print(f"✅ Всё загружено на: https://huggingface.co/{repo_id}")

test.py

@@ -0,0 +1,57 @@
+from transformers import MT5ForConditionalGeneration, MT5Tokenizer
+from datasets import load_dataset
+import torch
+
+# Путь к модели и данным
+model_path = "./mt5-finetuned"
+validation_file = "mt5_validation_data-1.jsonl"
+
+# Загрузка модели и токенизатора
+tokenizer = MT5Tokenizer.from_pretrained(model_path)
+model = MT5ForConditionalGeneration.from_pretrained(model_path)
+model.eval()
+
+# Используем GPU если есть
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = model.to(device)
+
+# Загрузка валидационной выборки
+dataset = load_dataset("json", data_files={"validation": validation_file})
+val_data = dataset["validation"]
+
+# Функция предсказания
+def predict(text):
+    inputs = tokenizer(text, return_tensors="pt", truncation=True, padding=True, max_length=256).to(device)
+    outputs = model.generate(
+        **inputs,
+        max_length=64,
+        num_beams=5,
+        early_stopping=True
+    )
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+# Подсчёт точности
+correct = 0
+results = []
+
+for idx, example in enumerate(val_data):
+    text = example["text"]
+    target = example["target"].strip()
+    pred = predict(text).strip()
+
+    results.append((text, pred, target))
+    if pred == target:
+        correct += 1
+
+# Примеры
+print("📋 Примеры предсказаний:\n")
+for i, (text, pred, target) in enumerate(results[:80]): # кол-во примеров 
+    print(f"#{i+1}")
+    print(f"📝 Вход: {text}")
+    print(f"✅ Target: {target}")
+    print(f"🤖 Предсказание: {pred}")
+    print("-" * 50)
+
+# Accuracy
+accuracy = correct / len(val_data)
+print(f"\n✅ Accuracy: {accuracy:.4f} ({correct}/{len(val_data)})\n")

train.py

@@ -0,0 +1,69 @@
+from transformers import MT5Tokenizer, MT5ForConditionalGeneration, Trainer, TrainingArguments
+from datasets import load_dataset
+import os
+import wandb
+
+#cd workspace && pip install --no-cache-dir -r requirements.txt
+#apt-get update && apt-get install -y screen & apt install git-lfs -y
+#screen -S train
+#python train.py
+
+# Загружаем модель и токенизатор
+model = MT5ForConditionalGeneration.from_pretrained("google/mt5-small")
+tokenizer = MT5Tokenizer.from_pretrained("google/mt5-small")
+
+# Загружаем датасет
+data_files = {
+    "train": "mt5_training_data-1.jsonl",
+    "validation": "mt5_validation_data-1.jsonl"
+}
+dataset = load_dataset("json", data_files=data_files)
+
+# Токенизация
+def tokenize_function(examples):
+    model_inputs = tokenizer(examples["text"], max_length=256, truncation=True, padding="max_length")
+    labels = tokenizer(examples["target"], max_length=64, truncation=True, padding="max_length")
+    model_inputs["labels"] = labels["input_ids"]
+    return model_inputs
+
+tokenized_datasets = dataset.map(tokenize_function, batched=True)
+
+wandb.login(key="5f028bc0142fb7fa45bdacdde3c00dbbaf8bf98e")
+
+training_args = TrainingArguments(
+    output_dir="./mt5-finetuned",
+    evaluation_strategy="steps",
+    eval_steps=100,
+    learning_rate=5e-5,
+    per_device_train_batch_size=250,
+    per_device_eval_batch_size=250,
+    num_train_epochs=3,
+    logging_steps=100,
+    warmup_ratio=0.06,
+    logging_first_step=True,
+    weight_decay=0.01,
+    logging_dir="./logs",
+    save_total_limit=2,
+    save_strategy="epoch",
+    report_to="wandb",                     
+    run_name="mt5-finetuning-run",   
+    disable_tqdm=False,
+    max_grad_norm=1.0                       
+)
+
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized_datasets["train"],
+    eval_dataset=tokenized_datasets["validation"]
+)
+
+# Обучение
+#trainer.train()
+trainer.train(resume_from_checkpoint=True)
+
+# Сохраняем локально
+model.save_pretrained("./mt5-finetuned")
+tokenizer.save_pretrained("./mt5-finetuned")
+print("✅ Модель сохранена локально в ./mt5-finetuned")