Training in progress, step 10, checkpoint

last-checkpoint/config.json

@@ -0,0 +1,50 @@
+{
+  "architectures": [
+    "OpenMythosForCausalLM"
+  ],
+  "bias": false,
+  "block_size": 256,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "effective_expected_depth": 10,
+  "eos_token_id": 2,
+  "head_dim": 64,
+  "init_values": {
+    "embed_scale": 16.0,
+    "embedding": 0.03952847075210474,
+    "out_proj": 0.008838834764831844,
+    "std": 0.03952847075210474
+  },
+  "injection_type": "lti",
+  "intermediate_size": 1024,
+  "kv_lora_rank": 512,
+  "mean_backprop_depth": 2,
+  "mean_recurrence": 4,
+  "model_type": "open_mythos",
+  "moe_intermediate_size": 1024,
+  "moe_top_k": 2,
+  "n_embd": 256,
+  "n_heads": 4,
+  "n_kv_heads": 4,
+  "n_layers": 4,
+  "n_layers_in_coda": 1,
+  "n_layers_in_prelude": 1,
+  "n_layers_in_recurrent_block": 2,
+  "n_routed_experts": 8,
+  "n_shared_experts": 2,
+  "norm_eps": 1e-06,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "q_lora_rank": 1536,
+  "qk_bias": true,
+  "rope_base": 50000.0,
+  "rope_head_dim": 64,
+  "state_init": "like-init",
+  "tie_embeddings": true,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.6.0",
+  "use_cache": false,
+  "use_mla": false,
+  "use_moe": false,
+  "vocab_size": 50257
+}

last-checkpoint/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.6.0"
+}

last-checkpoint/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cc1acb58f6552f17f47c9e79c248b5766ef129b9cc7004d0289960bcfa126ad
+size 68602152

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2812cf244d8d46428667984a8e6ab4a612874e817adebcbd079c4d0b057e17c
+size 137100235

last-checkpoint/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcd679242a831d4484be1b031fb6a525641a1324383532f6e1c8bb5ac52e4ce7
+size 14455

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f931434598425c3d35e696dc1ef9a0302880efc75ad45f5e1fbe43cfc68f080
+size 1465

last-checkpoint/trainer_state.json

@@ -0,0 +1,104 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 10,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.1,
+      "grad_norm": 3.904756546020508,
+      "learning_rate": 0.0,
+      "loss": 11.039416313171387,
+      "step": 1
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 2.952423572540283,
+      "learning_rate": 0.00015,
+      "loss": 11.029916763305664,
+      "step": 2
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 2.548621654510498,
+      "learning_rate": 0.0003,
+      "loss": 10.987221717834473,
+      "step": 3
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 2.724034309387207,
+      "learning_rate": 0.000288581929876693,
+      "loss": 10.806486129760742,
+      "step": 4
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 2.913846731185913,
+      "learning_rate": 0.00025606601717798207,
+      "loss": 10.382536888122559,
+      "step": 5
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 3.6747395992279053,
+      "learning_rate": 0.00020740251485476345,
+      "loss": 10.37621784210205,
+      "step": 6
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 2.790466785430908,
+      "learning_rate": 0.00015,
+      "loss": 10.06509780883789,
+      "step": 7
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 3.1298794746398926,
+      "learning_rate": 9.259748514523653e-05,
+      "loss": 10.32823371887207,
+      "step": 8
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 4.239963054656982,
+      "learning_rate": 4.3933982822017876e-05,
+      "loss": 9.877490997314453,
+      "step": 9
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 3.074774742126465,
+      "learning_rate": 1.1418070123306989e-05,
+      "loss": 9.85549545288086,
+      "step": 10
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 10,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 999999,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 131090350080.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e7be4b80f34efa8c6ffa736724ae3152fa29c25aa950c50b57dfee4c09cee52
+size 5329