End of training

README.md

@@ -14,7 +14,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9004
+- Loss: 1.2847
 
 ## Model description
 
@@ -33,32 +33,29 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-05
-- train_batch_size: 16
-- eval_batch_size: 16
+- learning_rate: 2e-05
+- train_batch_size: 64
+- eval_batch_size: 64
 - seed: 42
-- gradient_accumulation_steps: 2
-- total_train_batch_size: 32
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 512
 - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: linear
-- num_epochs: 1
+- num_epochs: 3
 - mixed_precision_training: Native AMP
 
 ### Training results
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 1.3546        | 0.1610 | 500  | 1.1776          |
-| 1.1799        | 0.3219 | 1000 | 1.0632          |
-| 1.1014        | 0.4829 | 1500 | 0.9904          |
-| 1.0385        | 0.6438 | 2000 | 0.9433          |
-| 1.0091        | 0.8048 | 2500 | 0.9097          |
-| 0.9824        | 0.9657 | 3000 | 0.9004          |
+| 2.2458        | 0.9996 | 854  | 2.1412          |
+| 1.5861        | 1.9996 | 1708 | 1.4378          |
+| 1.4097        | 2.9996 | 2562 | 1.2847          |
 
 
 ### Framework versions
 
-- Transformers 4.50.0
+- Transformers 4.50.3
 - Pytorch 2.6.0+cu124
 - Datasets 3.5.0
 - Tokenizers 0.21.1

config.json

@@ -1,30 +1 @@
-{
-  "architectures": [
-    "T5ForConditionalGeneration"
-  ],
-  "bos_token_id": 1,
-  "classifier_dropout": 0.0,
-  "d_ff": 2048,
-  "d_kv": 64,
-  "d_model": 512,
-  "decoder_start_token_id": 1,
-  "dense_act_fn": "relu",
-  "dropout_rate": 0.1,
-  "eos_token_id": 2,
-  "feed_forward_proj": "relu",
-  "initializer_factor": 1.0,
-  "is_encoder_decoder": true,
-  "is_gated_act": false,
-  "layer_norm_epsilon": 1e-06,
-  "model_type": "t5",
-  "num_decoder_layers": 6,
-  "num_heads": 8,
-  "num_layers": 6,
-  "pad_token_id": 0,
-  "relative_attention_max_distance": 128,
-  "relative_attention_num_buckets": 32,
-  "torch_dtype": "float32",
-  "transformers_version": "4.50.0",
-  "use_cache": true,
-  "vocab_size": 96
-}
+{"d_model": 256, "encoder_layers": 4, "decoder_layers": 4, "encoder_attention_heads": 8, "decoder_attention_heads": 8, "encoder_ffn_dim": 512, "decoder_ffn_dim": 512, "dropout": 0.1, "attention_dropout": 0.1, "activation_dropout": 0.1, "activation_function": "gelu", "max_position_embeddings": 128, "pad_token_id": 0, "bos_token_id": 1, "eos_token_id": 2, "use_cache": true, "centrality": false, "tf_idf": false, "vocab_size": 93}

generation_config.json

@@ -1,8 +1,9 @@
 {
   "_from_model_config": true,
   "bos_token_id": 1,
-  "decoder_start_token_id": 1,
+  "decoder_start_token_id": 2,
   "eos_token_id": 2,
+  "forced_eos_token_id": 2,
   "pad_token_id": 0,
-  "transformers_version": "4.50.0"
+  "transformers_version": "4.50.3"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:671e10ee98bf1730a8e65b6e389b8f4ec7c8c337fcdc8e3a321ca25344309d10
-size 176440352
+oid sha256:11021dd7d211ed75dc98f35dc671df526c476c1be61959604144a0985fdfe8aa
+size 21476748

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d14b9337946aebb9391884315c08c4831ab5e4c77d307a53492c38d50825b999
-size 5240
+oid sha256:d971ed7579fa5c5e7a8318adddc2162a389e76d385e75f108ecc818319f6f5e7
+size 5304

vocab.json

@@ -0,0 +1 @@
+{"char2id": {"<pad>": 0, "<s>": 1, "</s>": 2, "<unk>": 3, "\u0905": 4, "\u0906": 5, "\u0907": 6, "\u0908": 7, "\u0909": 8, "\u090a": 9, "\u090b": 10, "\u0960": 11, "\u090c": 12, "\u0961": 13, "\u090e": 14, "\u090f": 15, "\u0910": 16, "\u0913": 17, "\u0914": 18, "\u0915": 19, "\u0916": 20, "\u0917": 21, "\u0918": 22, "\u0919": 23, "\u091a": 24, "\u091b": 25, "\u091c": 26, "\u091d": 27, "\u091e": 28, "\u091f": 29, "\u0920": 30, "\u0921": 93, "\u0922": 101, "\u0923": 33, "\u0924": 34, "\u0925": 35, "\u0926": 36, "\u0927": 37, "\u0928": 38, "\u092a": 39, "\u092b": 99, "\u092c": 41, "\u092d": 42, "\u092e": 43, "\u092f": 44, "\u0930": 45, "\u0932": 46, "\u0935": 47, "\u0936": 48, "\u0937": 49, "\u0938": 50, "\u0939": 51, "\u094d": 52, "\u093e": 53, "\u093f": 54, "\u0940": 55, "\u0941": 56, "\u0942": 57, "\u0943": 58, "\u0944": 59, "\u0947": 60, "\u0948": 61, "\u094b": 62, "\u094c": 63, "\u0902": 86, "\u0903": 87, "\u0901": 67, "\u0964": 68, "\u0966": 69, "\u0967": 70, "\u0968": 71, "\u0969": 72, "\u096a": 73, "\u096b": 74, "\u096c": 75, "\u096d": 76, "\u096e": 77, "\u096f": 78, ",": 79, ".": 80, ";": 81, "?": 82, "!": 83, "\u093c": 102, "\u0945": 89, "\u094a": 88, "\u0912": 90, "\u093d": 91, "\u0949": 92, "\u0950": 95, "\u0965": 96, "\u0911": 97, "\u0931": 98}, "id2char": {"0": "<pad>", "1": "<s>", "2": "</s>", "3": "<unk>", "4": "\u0905", "5": "\u0906", "6": "\u0907", "7": "\u0908", "8": "\u0909", "9": "\u090a", "10": "\u090b", "11": "\u0960", "12": "\u090c", "13": "\u0961", "14": "\u090e", "15": "\u090f", "16": "\u0910", "17": "\u0913", "18": "\u0914", "19": "\u0915", "20": "\u0916", "21": "\u0917", "22": "\u0918", "23": "\u0919", "24": "\u091a", "25": "\u091b", "26": "\u091c", "27": "\u091d", "28": "\u091e", "29": "\u091f", "30": "\u0920", "93": "\u0921", "101": "\u0922", "33": "\u0923", "34": "\u0924", "35": "\u0925", "36": "\u0926", "37": "\u0927", "38": "\u0928", "39": "\u092a", "99": "\u092b", "41": "\u092c", "42": "\u092d", "43": "\u092e", "44": "\u092f", "45": "\u0930", "46": "\u0932", "47": "\u0935", "48": "\u0936", "49": "\u0937", "50": "\u0938", "51": "\u0939", "52": "\u094d", "53": "\u093e", "54": "\u093f", "55": "\u0940", "56": "\u0941", "57": "\u0942", "58": "\u0943", "59": "\u0944", "60": "\u0947", "61": "\u0948", "62": "\u094b", "63": "\u094c", "86": "\u0902", "87": "\u0903", "67": "\u0901", "68": "\u0964", "69": "\u0966", "70": "\u0967", "71": "\u0968", "72": "\u0969", "73": "\u096a", "74": "\u096b", "75": "\u096c", "76": "\u096d", "77": "\u096e", "78": "\u096f", "79": ",", "80": ".", "81": ";", "82": "?", "83": "!", "102": "\u093c", "89": "\u0945", "88": "\u094a", "90": "\u0912", "91": "\u093d", "92": "\u0949", "95": "\u0950", "96": "\u0965", "97": "\u0911", "98": "\u0931"}}