Training in progress, step 10

config.json

@@ -1,26 +1,21 @@
 {
-  "_name_or_path": "ccdv/lsg-bart-base-4096-arxiv",
-  "activation_dropout": 0.1,
+  "_name_or_path": "allenai/led-base-16384",
+  "activation_dropout": 0.0,
   "activation_function": "gelu",
-  "adaptive": true,
-  "add_bias_logits": false,
-  "add_final_layer_norm": false,
   "architectures": [
-    "LSGBartForConditionalGeneration"
+    "LEDForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "attention_window": [
+    1024,
+    1024,
+    1024,
+    1024,
+    1024,
+    1024
   ],
-  "attention_dropout": 0.1,
-  "auto_map": {
-    "AutoConfig": "modeling_lsg_bart.LSGBartConfig",
-    "AutoModel": "modeling_lsg_bart.LSGBartModel",
-    "AutoModelForCausalLM": "modeling_lsg_bart.LSGBartForCausalLM",
-    "AutoModelForQuestionAnswering": "modeling_lsg_bart.LSGBartForQuestionAnswering",
-    "AutoModelForSeq2SeqLM": "modeling_lsg_bart.LSGBartForConditionalGeneration",
-    "AutoModelForSequenceClassification": "modeling_lsg_bart.LSGBartForSequenceClassification"
-  },
-  "base_model_prefix": "lsg",
-  "block_size": 256,
   "bos_token_id": 0,
-  "classif_dropout": 0.1,
+  "classif_dropout": 0.0,
   "classifier_dropout": 0.0,
   "d_model": 768,
   "decoder_attention_heads": 12,
@@ -29,14 +24,11 @@
   "decoder_layers": 6,
   "decoder_start_token_id": 2,
   "dropout": 0.1,
-  "early_stopping": true,
   "encoder_attention_heads": 12,
   "encoder_ffn_dim": 3072,
   "encoder_layerdrop": 0.0,
   "encoder_layers": 6,
   "eos_token_id": 2,
-  "forced_bos_token_id": 0,
-  "forced_eos_token_id": 2,
   "gradient_checkpointing": false,
   "id2label": {
     "0": "LABEL_0",
@@ -50,46 +42,11 @@
     "LABEL_1": 1,
     "LABEL_2": 2
   },
-  "length_penalty": 2.0,
-  "lsh_num_pre_rounds": 1,
-  "mask_first_token": false,
-  "max_length": 320,
-  "max_position_embeddings": 4096,
-  "min_length": 64,
-  "model_type": "bart",
-  "no_repeat_ngram_size": null,
-  "normalize_before": false,
-  "normalize_embedding": true,
-  "num_beams": 5,
-  "num_global_tokens": 1,
+  "max_decoder_position_embeddings": 1024,
+  "max_encoder_position_embeddings": 16384,
+  "model_type": "led",
   "num_hidden_layers": 6,
   "pad_token_id": 1,
-  "pass_global_tokens_to_decoder": true,
-  "pool_with_global": true,
-  "scale_embedding": false,
-  "sparse_block_size": 0,
-  "sparsity_factor": 2,
-  "sparsity_type": "none",
-  "task_specific_params": {
-    "summarization": {
-      "length_penalty": 1.0,
-      "max_length": 128,
-      "min_length": 12,
-      "num_beams": 4
-    },
-    "summarization_cnn": {
-      "length_penalty": 2.0,
-      "max_length": 142,
-      "min_length": 56,
-      "num_beams": 4
-    },
-    "summarization_xsum": {
-      "length_penalty": 1.0,
-      "max_length": 62,
-      "min_length": 11,
-      "num_beams": 6
-    }
-  },
   "torch_dtype": "float32",
   "transformers_version": "4.38.2",
   "use_cache": true,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76e155607c88146e4c2ec60c218d1a24b0034f2a7d6946a49fa155921f7f853b
-size 578359980
+oid sha256:2c627f2d7f76e9568520731512814557b46bd31caed2b1526517aad7885116f1
+size 647614116

runs/Apr17_14-49-50_d95edb3f4728/events.out.tfevents.1713365810.d95edb3f4728.1521.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee41b6750904b1368198b29fe2040d1ae2a83caac6da5333734b30961d5b403b
+size 5227

runs/Apr17_15-08-33_d95edb3f4728/events.out.tfevents.1713366520.d95edb3f4728.1521.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a11b591e6620a87d80d2897f438e27a115f5195977f2d903c2775b5f274d193b
+size 5227

runs/Apr17_15-10-36_d95edb3f4728/events.out.tfevents.1713366637.d95edb3f4728.1521.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:529a5dcf01cef883be146acf525a07d7f98ebdbd1942ef24c251399eca9c9ffa
+size 5227

runs/Apr17_15-13-55_d95edb3f4728/events.out.tfevents.1713366836.d95edb3f4728.1521.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a488955c92587c4d6f2d75e3a548f2db94e7d429a42366cf9bdbeed18c23663
+size 5742

special_tokens_map.json

@@ -2,49 +2,49 @@
   "bos_token": {
     "content": "<s>",
     "lstrip": false,
-    "normalized": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },
   "cls_token": {
     "content": "<s>",
     "lstrip": false,
-    "normalized": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
     "content": "</s>",
     "lstrip": false,
-    "normalized": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },
   "mask_token": {
     "content": "<mask>",
     "lstrip": true,
-    "normalized": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
     "content": "<pad>",
     "lstrip": false,
-    "normalized": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },
   "sep_token": {
     "content": "</s>",
     "lstrip": false,
-    "normalized": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,
-    "normalized": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   }

tokenizer.json

@@ -2,11 +2,20 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 512,
+    "max_length": 128,
     "strategy": "LongestFirst",
     "stride": 0
   },
-  "padding": null,
+  "padding": {
+    "strategy": {
+      "Fixed": 128
+    },
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 1,
+    "pad_type_id": 0,
+    "pad_token": "<pad>"
+  },
   "added_tokens": [
     {
       "id": 0,
@@ -14,7 +23,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     },
     {
@@ -23,7 +32,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     },
     {
@@ -32,7 +41,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     },
     {
@@ -41,7 +50,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     },
     {
@@ -50,7 +59,7 @@
       "single_word": false,
       "lstrip": true,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     }
   ],

tokenizer_config.json

@@ -4,7 +4,7 @@
     "0": {
       "content": "<s>",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
@@ -12,7 +12,7 @@
     "1": {
       "content": "<pad>",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
@@ -20,7 +20,7 @@
     "2": {
       "content": "</s>",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
@@ -28,7 +28,7 @@
     "3": {
       "content": "<unk>",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
@@ -36,7 +36,7 @@
     "50264": {
       "content": "<mask>",
       "lstrip": true,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
@@ -48,17 +48,10 @@
   "eos_token": "</s>",
   "errors": "replace",
   "mask_token": "<mask>",
-  "max_length": 4096,
-  "model_max_length": 4096,
-  "pad_to_multiple_of": null,
+  "model_max_length": 16384,
   "pad_token": "<pad>",
-  "pad_token_type_id": 0,
-  "padding_side": "right",
   "sep_token": "</s>",
-  "stride": 0,
-  "tokenizer_class": "BartTokenizer",
+  "tokenizer_class": "LEDTokenizer",
   "trim_offsets": true,
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db18f804e2bd20316119d1ffa79505edcc276e8ff3c85a0b55e54e7fac31264c
-size 578214938
+oid sha256:b04ebab9a3ba6cd92ac3125e23c05617fd037b606f023000f3ccbe603cf9c5eb
+size 5048