Hourly training update [skip ci]

checkpoint/config.json

@@ -1,44 +1,44 @@
-{
-  "architectures": [
-    "LlamaForCausalLM"
-  ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "bos_token_id": 1,
-  "capacity_factor_eval": 2.0,
-  "capacity_factor_train": 1.25,
-  "drop_tks": true,
-  "dropout_rate": 0.0,
-  "dtype": "float32",
-  "eos_token_id": 2,
-  "expert_parallel": null,
-  "gated": true,
-  "head_dim": 64,
-  "hidden_act": "silu",
-  "hidden_size": 768,
-  "initializer_range": 0.02,
-  "intermediate_size": 2048,
-  "layer_norm_epsilon": 1e-06,
-  "max_position_embeddings": 2048,
-  "min_capacity": 4,
-  "mlp_bias": false,
-  "model_type": "llama",
-  "moe_layer_interval": 4,
-  "noisy_policy": null,
-  "num_attention_heads": 12,
-  "num_experts": 16,
-  "num_hidden_layers": 12,
-  "num_key_value_heads": 12,
-  "pad_token_id": 0,
-  "pretraining_tp": 1,
-  "rms_norm_eps": 1e-06,
-  "rope_parameters": {
-    "rope_theta": 10000.0,
-    "rope_type": "default"
-  },
-  "tie_word_embeddings": false,
-  "topk": 2,
-  "transformers_version": "5.2.0",
-  "use_cache": true,
-  "vocab_size": 256384
-}
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "capacity_factor_eval": 2.0,
+  "capacity_factor_train": 1.25,
+  "drop_tks": true,
+  "dropout_rate": 0.0,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expert_parallel": null,
+  "gated": true,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "layer_norm_epsilon": 1e-06,
+  "max_position_embeddings": 2048,
+  "min_capacity": 4,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "moe_layer_interval": 4,
+  "noisy_policy": null,
+  "num_attention_heads": 12,
+  "num_experts": 16,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 12,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "topk": 2,
+  "transformers_version": "5.2.0",
+  "use_cache": true,
+  "vocab_size": 256384
+}

checkpoint/dataset_state.json

@@ -0,0 +1 @@
+{"processed_items": 18050}

checkpoint/ewc_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a3d95dfbe5b1c010ded2ad3f6514d6278afe76ef527276718ac9129ec5f504b
+size 1915110091

checkpoint/generation_config.json

@@ -1,7 +1,7 @@
-{
-  "_from_model_config": true,
-  "bos_token_id": 1,
-  "eos_token_id": 2,
-  "pad_token_id": 0,
-  "transformers_version": "5.2.0"
-}
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
+  "transformers_version": "5.2.0"
+}

checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f3f03d39b91def519f25bf10cf082186b0539034123df3d1a89cb3b1bb58370c
+oid sha256:3f22effc917fdcec6e0f390cc7119c9a0a69d32586ab8d92355c991b60c755eb
 size 1915051264

checkpoint/tokenizer.json

@@ -1,7 +1,21 @@
 {
   "version": "1.0",
-  "truncation": null,
-  "padding": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 512,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
+  "padding": {
+    "strategy": {
+      "Fixed": 512
+    },
+    "direction": "Left",
+    "pad_to_multiple_of": null,
+    "pad_id": 2,
+    "pad_type_id": 0,
+    "pad_token": "</s>"
+  },
   "added_tokens": [
     {
       "id": 0,

checkpoint/tokenizer_config.json

@@ -1,25 +1,25 @@
-{
-  "add_prefix_space": true,
-  "backend": "tokenizers",
-  "bos_token": "<s>",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "</s>",
-  "eot_token": "▁<EOT>",
-  "extra_special_tokens": [
-    "▁<PRE>",
-    "▁<MID>",
-    "▁<SUF>",
-    "▁<EOT>",
-    "<FILL_ME>"
-  ],
-  "fill_token": "<FILL_ME>",
-  "is_local": false,
-  "middle_token": "▁<MID>",
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "</s>",
-  "prefix_token": "▁<PRE>",
-  "suffix_token": "▁<SUF>",
-  "tokenizer_class": "CodeLlamaTokenizer",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false
-}
+{
+  "add_prefix_space": true,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "eot_token": "▁<EOT>",
+  "extra_special_tokens": [
+    "▁<PRE>",
+    "▁<MID>",
+    "▁<SUF>",
+    "▁<EOT>",
+    "<FILL_ME>"
+  ],
+  "fill_token": "<FILL_ME>",
+  "is_local": false,
+  "middle_token": "▁<MID>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "prefix_token": "▁<PRE>",
+  "suffix_token": "▁<SUF>",
+  "tokenizer_class": "CodeLlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint/trainer_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43d21d1b58a7f16e549213785845726d53dbef75eb134b24bdad633cb8387137
+size 1361