napatswift/mt5-bud-seq2seq

README.md

@@ -1,6 +1,6 @@
 ---
 license: apache-2.0
-base_model: napatswift/mt5-fixth
+base_model: google/flan-t5-base
 tags:
 - generated_from_trainer
 model-index:
@@ -13,14 +13,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # mt5-budget-seq
 
-This model is a fine-tuned version of [napatswift/mt5-fixth](https://huggingface.co/napatswift/mt5-fixth) on an unknown dataset.
+This model is a fine-tuned version of [google/flan-t5-base](https://huggingface.co/google/flan-t5-base) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- eval_loss: 0.0390
-- eval_runtime: 6.1075
-- eval_samples_per_second: 6.549
-- eval_steps_per_second: 6.549
-- epoch: 1.65
-- step: 402
+- Loss: 0.0238
 
 ## Model description
 
@@ -40,15 +35,22 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0005
-- train_batch_size: 1
+- train_batch_size: 6
 - eval_batch_size: 1
 - seed: 42
 - gradient_accumulation_steps: 16
-- total_train_batch_size: 16
+- total_train_batch_size: 96
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_steps: 10
-- num_epochs: 10
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 0.1045        | 3.3   | 201  | 0.0238          |
+
 
 ### Framework versions
 

added_tokens.json

@@ -1,25 +1,25 @@
 {
-  "</amount>": 250100,
-  "</fiscal_year>": 250101,
-  "</fiscal_year_end_year>": 250102,
-  "</fiscal_year_year>": 250103,
-  "</line_item_amount>": 250104,
-  "</line_item_bullet>": 250105,
-  "</line_item_children>": 250106,
-  "</line_item_fiscal_years>": 250107,
-  "</line_item_name>": 250108,
-  "</line_item_root>": 250109,
-  "<amount>": 250110,
-  "<begin>": 250111,
-  "<begin_indent>": 250112,
-  "<big_sep>": 250113,
-  "<fiscal_year>": 250114,
-  "<fiscal_year_end_year>": 250115,
-  "<fiscal_year_year>": 250116,
-  "<line_item_amount>": 250117,
-  "<line_item_bullet>": 250118,
-  "<line_item_children>": 250119,
-  "<line_item_fiscal_years>": 250120,
-  "<line_item_name>": 250121,
-  "<line_item_root>": 250122
+  "</amount>": 32100,
+  "</fiscal_year>": 32101,
+  "</fiscal_year_end_year>": 32102,
+  "</fiscal_year_year>": 32103,
+  "</line_item_amount>": 32104,
+  "</line_item_bullet>": 32105,
+  "</line_item_children>": 32106,
+  "</line_item_fiscal_years>": 32107,
+  "</line_item_name>": 32108,
+  "</line_item_root>": 32109,
+  "<amount>": 32110,
+  "<begin>": 32111,
+  "<begin_indent>": 32112,
+  "<big_sep>": 32113,
+  "<fiscal_year>": 32114,
+  "<fiscal_year_end_year>": 32115,
+  "<fiscal_year_year>": 32116,
+  "<line_item_amount>": 32117,
+  "<line_item_bullet>": 32118,
+  "<line_item_children>": 32119,
+  "<line_item_fiscal_years>": 32120,
+  "<line_item_name>": 32121,
+  "<line_item_root>": 32122
 }

config.json

@@ -1,12 +1,12 @@
 {
-  "_name_or_path": "napatswift/mt5-fixth",
+  "_name_or_path": "google/flan-t5-base",
   "architectures": [
-    "MT5ForConditionalGeneration"
+    "T5ForConditionalGeneration"
   ],
   "classifier_dropout": 0.0,
-  "d_ff": 1024,
+  "d_ff": 2048,
   "d_kv": 64,
-  "d_model": 512,
+  "d_model": 768,
   "decoder_start_token_id": 0,
   "dense_act_fn": "gelu_new",
   "dropout_rate": 0.1,
@@ -16,17 +16,47 @@
   "is_encoder_decoder": true,
   "is_gated_act": true,
   "layer_norm_epsilon": 1e-06,
-  "model_type": "mt5",
-  "num_decoder_layers": 8,
-  "num_heads": 6,
-  "num_layers": 8,
+  "model_type": "t5",
+  "n_positions": 512,
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
   "pad_token_id": 0,
   "relative_attention_max_distance": 128,
   "relative_attention_num_buckets": 32,
+  "task_specific_params": {
+    "summarization": {
+      "early_stopping": true,
+      "length_penalty": 2.0,
+      "max_length": 200,
+      "min_length": 30,
+      "no_repeat_ngram_size": 3,
+      "num_beams": 4,
+      "prefix": "summarize: "
+    },
+    "translation_en_to_de": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to German: "
+    },
+    "translation_en_to_fr": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to French: "
+    },
+    "translation_en_to_ro": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to Romanian: "
+    }
+  },
   "tie_word_embeddings": false,
-  "tokenizer_class": "T5Tokenizer",
   "torch_dtype": "float32",
   "transformers_version": "4.33.3",
   "use_cache": true,
-  "vocab_size": 250124
+  "vocab_size": 32124
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9de51b6fccaa4cf2dcc1565ba777211e82c88fce4b102c7910dd132213a1df6b
-size 1200821765
+oid sha256:9746636c9f61901a70733adb8f6087ce4df3825eaa6963811bcf6c74babf6c96
+size 990384309

special_tokens_map.json

@@ -1,4 +1,106 @@
 {
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
   "eos_token": "</s>",
   "pad_token": "<pad>",
   "unk_token": "<unk>"

spiece.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
-size 4309802
+oid sha256:d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86
+size 791656

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ded8087ed38e31d69b670456595183f4efca67fa94df4653c70fa5a7017ebee6
-size 16319677
+oid sha256:e58c720ed83af78e2768b2cfa582aa15ea75630b56f0181045194419e7dbd805
+size 2426696

tokenizer_config.json

@@ -1,16 +1,112 @@
 {
-  "additional_special_tokens": null,
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
   "clean_up_tokenization_spaces": true,
   "eos_token": "</s>",
-  "extra_ids": 0,
-  "legacy": true,
-  "max_length": 512,
-  "model_max_length": 1000000000000000019884624838656,
+  "extra_ids": 100,
+  "model_max_length": 512,
   "pad_token": "<pad>",
   "sp_model_kwargs": {},
-  "stride": 0,
   "tokenizer_class": "T5Tokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9711c5d3d0a69c972010153b7313e4f8ca0f06ec7916149b1716fc68c228220
-size 4155
+oid sha256:b8f98faa36e77f999a1860f54d527d0e79bff674283d7be7e59d075c46fa42df
+size 4091