Training in progress, step 500, checkpoint

last-checkpoint/config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5504,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 16,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

last-checkpoint/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.55.0.dev0"
+}

last-checkpoint/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75b0e3e494d3c97bf01c086fa4e5f6f04a62cf1680cc25727492d425666eed04
+size 4984282912

last-checkpoint/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1a622e7dd405333344ea67affa6f3ea99eaddc628a940a1b6f01f5854dd5241
+size 397435656

last-checkpoint/model.safetensors.index.json

@@ -0,0 +1,227 @@
+{
+  "metadata": {
+    "total_parameters": 1345423360,
+    "total_size": 5381693440
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35ab7e0e182b23df13f1b93f19f4a2fffaf86b7f128bdfb786814dfd72eb554e
+size 10763577383

last-checkpoint/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645

last-checkpoint/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3493c47af98b05be537607476477856b71d1034b6ccd558729989cb227a25a9
+size 1383

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dc9aad46f12a6921d4c23c3fe5f84001eaed6c0b51a83c6733dc49db400dc31
+size 1465

last-checkpoint/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

last-checkpoint/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

last-checkpoint/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

last-checkpoint/trainer_state.json

@@ -0,0 +1,384 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.007050992779783393,
+  "eval_steps": 500,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00014101985559566787,
+      "grad_norm": 3.8230981826782227,
+      "learning_rate": 4.999811973525873e-05,
+      "loss": 9.6311,
+      "step": 10
+    },
+    {
+      "epoch": 0.00028203971119133574,
+      "grad_norm": 3.3901870250701904,
+      "learning_rate": 4.9995769404332135e-05,
+      "loss": 8.465,
+      "step": 20
+    },
+    {
+      "epoch": 0.0004230595667870036,
+      "grad_norm": 2.0604257583618164,
+      "learning_rate": 4.999341907340554e-05,
+      "loss": 7.9593,
+      "step": 30
+    },
+    {
+      "epoch": 0.0005640794223826715,
+      "grad_norm": 2.740540027618408,
+      "learning_rate": 4.999106874247894e-05,
+      "loss": 7.7688,
+      "step": 40
+    },
+    {
+      "epoch": 0.0007050992779783394,
+      "grad_norm": 1.9339054822921753,
+      "learning_rate": 4.998871841155235e-05,
+      "loss": 7.5742,
+      "step": 50
+    },
+    {
+      "epoch": 0.0008461191335740072,
+      "grad_norm": 1.5775341987609863,
+      "learning_rate": 4.998636808062576e-05,
+      "loss": 7.5567,
+      "step": 60
+    },
+    {
+      "epoch": 0.000987138989169675,
+      "grad_norm": 1.8233592510223389,
+      "learning_rate": 4.998401774969916e-05,
+      "loss": 7.4125,
+      "step": 70
+    },
+    {
+      "epoch": 0.001128158844765343,
+      "grad_norm": 2.22709059715271,
+      "learning_rate": 4.998166741877257e-05,
+      "loss": 7.442,
+      "step": 80
+    },
+    {
+      "epoch": 0.0012691787003610109,
+      "grad_norm": 2.862053632736206,
+      "learning_rate": 4.9979317087845974e-05,
+      "loss": 7.3451,
+      "step": 90
+    },
+    {
+      "epoch": 0.0014101985559566788,
+      "grad_norm": 2.302476167678833,
+      "learning_rate": 4.9976966756919375e-05,
+      "loss": 7.2534,
+      "step": 100
+    },
+    {
+      "epoch": 0.0015512184115523465,
+      "grad_norm": 2.0931568145751953,
+      "learning_rate": 4.997461642599278e-05,
+      "loss": 7.2254,
+      "step": 110
+    },
+    {
+      "epoch": 0.0016922382671480144,
+      "grad_norm": 1.8261802196502686,
+      "learning_rate": 4.997226609506619e-05,
+      "loss": 7.1342,
+      "step": 120
+    },
+    {
+      "epoch": 0.0018332581227436823,
+      "grad_norm": 1.8617385625839233,
+      "learning_rate": 4.996991576413959e-05,
+      "loss": 7.07,
+      "step": 130
+    },
+    {
+      "epoch": 0.00197427797833935,
+      "grad_norm": 2.7089622020721436,
+      "learning_rate": 4.9967565433213e-05,
+      "loss": 6.9381,
+      "step": 140
+    },
+    {
+      "epoch": 0.002115297833935018,
+      "grad_norm": 2.621422290802002,
+      "learning_rate": 4.9965215102286406e-05,
+      "loss": 6.8489,
+      "step": 150
+    },
+    {
+      "epoch": 0.002256317689530686,
+      "grad_norm": 2.1687350273132324,
+      "learning_rate": 4.996286477135981e-05,
+      "loss": 6.8548,
+      "step": 160
+    },
+    {
+      "epoch": 0.0023973375451263536,
+      "grad_norm": 3.110124111175537,
+      "learning_rate": 4.9960514440433215e-05,
+      "loss": 6.8635,
+      "step": 170
+    },
+    {
+      "epoch": 0.0025383574007220217,
+      "grad_norm": 1.9620782136917114,
+      "learning_rate": 4.995816410950662e-05,
+      "loss": 6.7232,
+      "step": 180
+    },
+    {
+      "epoch": 0.0026793772563176894,
+      "grad_norm": 2.152229070663452,
+      "learning_rate": 4.995581377858002e-05,
+      "loss": 6.7775,
+      "step": 190
+    },
+    {
+      "epoch": 0.0028203971119133576,
+      "grad_norm": 1.85105299949646,
+      "learning_rate": 4.995346344765343e-05,
+      "loss": 6.5831,
+      "step": 200
+    },
+    {
+      "epoch": 0.0029614169675090253,
+      "grad_norm": 2.4505670070648193,
+      "learning_rate": 4.995111311672684e-05,
+      "loss": 6.5,
+      "step": 210
+    },
+    {
+      "epoch": 0.003102436823104693,
+      "grad_norm": 2.475897789001465,
+      "learning_rate": 4.994876278580024e-05,
+      "loss": 6.552,
+      "step": 220
+    },
+    {
+      "epoch": 0.003243456678700361,
+      "grad_norm": 2.4876480102539062,
+      "learning_rate": 4.9946412454873646e-05,
+      "loss": 6.3951,
+      "step": 230
+    },
+    {
+      "epoch": 0.003384476534296029,
+      "grad_norm": 2.7807083129882812,
+      "learning_rate": 4.9944062123947054e-05,
+      "loss": 6.5027,
+      "step": 240
+    },
+    {
+      "epoch": 0.0035254963898916965,
+      "grad_norm": 2.436201810836792,
+      "learning_rate": 4.9941711793020455e-05,
+      "loss": 6.4606,
+      "step": 250
+    },
+    {
+      "epoch": 0.0036665162454873647,
+      "grad_norm": 3.3593878746032715,
+      "learning_rate": 4.993936146209386e-05,
+      "loss": 6.2834,
+      "step": 260
+    },
+    {
+      "epoch": 0.0038075361010830324,
+      "grad_norm": 1.8510562181472778,
+      "learning_rate": 4.993701113116727e-05,
+      "loss": 6.3189,
+      "step": 270
+    },
+    {
+      "epoch": 0.0039485559566787,
+      "grad_norm": 2.0974020957946777,
+      "learning_rate": 4.993466080024068e-05,
+      "loss": 6.2319,
+      "step": 280
+    },
+    {
+      "epoch": 0.004089575812274368,
+      "grad_norm": 2.1588592529296875,
+      "learning_rate": 4.993231046931408e-05,
+      "loss": 6.2977,
+      "step": 290
+    },
+    {
+      "epoch": 0.004230595667870036,
+      "grad_norm": 2.1286563873291016,
+      "learning_rate": 4.9929960138387486e-05,
+      "loss": 5.9808,
+      "step": 300
+    },
+    {
+      "epoch": 0.004371615523465704,
+      "grad_norm": 2.310302495956421,
+      "learning_rate": 4.9927609807460894e-05,
+      "loss": 6.1973,
+      "step": 310
+    },
+    {
+      "epoch": 0.004512635379061372,
+      "grad_norm": 1.9249286651611328,
+      "learning_rate": 4.9925259476534294e-05,
+      "loss": 6.1088,
+      "step": 320
+    },
+    {
+      "epoch": 0.00465365523465704,
+      "grad_norm": 2.6528685092926025,
+      "learning_rate": 4.99229091456077e-05,
+      "loss": 6.1837,
+      "step": 330
+    },
+    {
+      "epoch": 0.004794675090252707,
+      "grad_norm": 2.3377368450164795,
+      "learning_rate": 4.992055881468111e-05,
+      "loss": 5.8968,
+      "step": 340
+    },
+    {
+      "epoch": 0.004935694945848375,
+      "grad_norm": 2.1544837951660156,
+      "learning_rate": 4.991820848375452e-05,
+      "loss": 6.0234,
+      "step": 350
+    },
+    {
+      "epoch": 0.0050767148014440435,
+      "grad_norm": 2.352724075317383,
+      "learning_rate": 4.991585815282792e-05,
+      "loss": 5.9803,
+      "step": 360
+    },
+    {
+      "epoch": 0.005217734657039712,
+      "grad_norm": 2.082303047180176,
+      "learning_rate": 4.9913507821901325e-05,
+      "loss": 5.8347,
+      "step": 370
+    },
+    {
+      "epoch": 0.005358754512635379,
+      "grad_norm": 2.1395177841186523,
+      "learning_rate": 4.991115749097473e-05,
+      "loss": 5.7546,
+      "step": 380
+    },
+    {
+      "epoch": 0.005499774368231047,
+      "grad_norm": 2.0028514862060547,
+      "learning_rate": 4.990880716004814e-05,
+      "loss": 5.8794,
+      "step": 390
+    },
+    {
+      "epoch": 0.005640794223826715,
+      "grad_norm": 2.101605176925659,
+      "learning_rate": 4.990645682912155e-05,
+      "loss": 5.7707,
+      "step": 400
+    },
+    {
+      "epoch": 0.0057818140794223824,
+      "grad_norm": 2.373682737350464,
+      "learning_rate": 4.990410649819495e-05,
+      "loss": 5.7208,
+      "step": 410
+    },
+    {
+      "epoch": 0.005922833935018051,
+      "grad_norm": 2.3248589038848877,
+      "learning_rate": 4.9901756167268357e-05,
+      "loss": 5.7167,
+      "step": 420
+    },
+    {
+      "epoch": 0.006063853790613719,
+      "grad_norm": 2.1718509197235107,
+      "learning_rate": 4.9899405836341764e-05,
+      "loss": 5.6323,
+      "step": 430
+    },
+    {
+      "epoch": 0.006204873646209386,
+      "grad_norm": 1.9086527824401855,
+      "learning_rate": 4.9897055505415165e-05,
+      "loss": 5.6538,
+      "step": 440
+    },
+    {
+      "epoch": 0.006345893501805054,
+      "grad_norm": 2.4224274158477783,
+      "learning_rate": 4.989470517448857e-05,
+      "loss": 5.7213,
+      "step": 450
+    },
+    {
+      "epoch": 0.006486913357400722,
+      "grad_norm": 2.1714367866516113,
+      "learning_rate": 4.989235484356198e-05,
+      "loss": 5.6312,
+      "step": 460
+    },
+    {
+      "epoch": 0.0066279332129963895,
+      "grad_norm": 1.9736969470977783,
+      "learning_rate": 4.989000451263538e-05,
+      "loss": 5.5375,
+      "step": 470
+    },
+    {
+      "epoch": 0.006768953068592058,
+      "grad_norm": 2.426051139831543,
+      "learning_rate": 4.988765418170879e-05,
+      "loss": 5.5596,
+      "step": 480
+    },
+    {
+      "epoch": 0.006909972924187726,
+      "grad_norm": 1.9649548530578613,
+      "learning_rate": 4.9885303850782196e-05,
+      "loss": 5.6028,
+      "step": 490
+    },
+    {
+      "epoch": 0.007050992779783393,
+      "grad_norm": 2.743234634399414,
+      "learning_rate": 4.98829535198556e-05,
+      "loss": 5.6177,
+      "step": 500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 212736,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7863627939840000.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e8f62f1a7f9de9b42aace6eb1ce1d04aab661da501089522b68d255da01cb9c
+size 5777