Training in progress, step 1000, checkpoint

last-checkpoint/config.json

@@ -0,0 +1,48 @@
+{
+  "architectures": [
+    "SmalLmForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.1,
+  "balancing_coef": 0.0001,
+  "bos_token_id": 1,
+  "embedding_dropout": 0.0,
+  "eos_token_id": 0,
+  "expert_size": 576,
+  "gate_noise": false,
+  "head_size": 72,
+  "hidden_size": 576,
+  "high_rotations": 32,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "layer_dropout": 0.1,
+  "low_rotations": 1,
+  "max_seq_len": 1024,
+  "mlp_bias": false,
+  "model_type": "smallm",
+  "moe_bias": false,
+  "moe_period": 2,
+  "no_moe_layers": 10,
+  "noisy_experts": false,
+  "num_attention_heads": 8,
+  "num_hidden_layers": 26,
+  "num_kv_heads": 4,
+  "original_seq_len": 1024,
+  "pad_token_id": 0,
+  "positional_bias_type": "rope",
+  "rms_affine": false,
+  "rms_norm_eps": 1e-06,
+  "rope_base": 100000,
+  "routed_experts": 8,
+  "shared_experts": 1,
+  "sliding_window_attention": true,
+  "sliding_window_context": 1024,
+  "sliding_window_period": 4,
+  "static_residual": false,
+  "token_experts": 3,
+  "torch_dtype": "float32",
+  "transformers_version": "4.50.3",
+  "use_cache": true,
+  "use_moe": false,
+  "vocab_size": 60000
+}

last-checkpoint/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 0,
+  "pad_token_id": 0,
+  "transformers_version": "4.50.3"
+}

last-checkpoint/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f297721cb03be26e6092587489bee99025182be771eea6e05f77edb6bd34f03
+size 517931840

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6081c2cece2bea94b0bdedc301e6d7a3c34eef0aaba0899c0043359233ba4468
+size 1035661434

last-checkpoint/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b49390de6348da607ac8b8e7ddf69e26d2c764b165fb5c92780c29f0de564e6c
+size 14244

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78e83e408c816bc1f16e9a76bc9d7e37229ddececd6ea07286eb16c742dca118
+size 1064

last-checkpoint/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|beginoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

last-checkpoint/tokenizer_config.json

@@ -0,0 +1,115 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|beginoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|reserved_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|reserved_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<|reserved_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<|reserved_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<|reserved_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<|reserved_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<|reserved_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<|reserved_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<|reserved_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<|reserved_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<|reserved_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|beginoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}

last-checkpoint/trainer_state.json

@@ -0,0 +1,212 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.02196604324290188,
+  "eval_steps": 500,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.001098302162145094,
+      "grad_norm": 1.8242720365524292,
+      "learning_rate": 1e-05,
+      "loss": 10.2008,
+      "num_input_tokens_seen": 52428800,
+      "step": 50
+    },
+    {
+      "epoch": 0.002196604324290188,
+      "grad_norm": 1.7588739395141602,
+      "learning_rate": 2e-05,
+      "loss": 9.6579,
+      "num_input_tokens_seen": 104857600,
+      "step": 100
+    },
+    {
+      "epoch": 0.003294906486435282,
+      "grad_norm": 1.4990028142929077,
+      "learning_rate": 3e-05,
+      "loss": 8.9277,
+      "num_input_tokens_seen": 157286400,
+      "step": 150
+    },
+    {
+      "epoch": 0.004393208648580376,
+      "grad_norm": 0.9206030368804932,
+      "learning_rate": 4e-05,
+      "loss": 8.0739,
+      "num_input_tokens_seen": 209715200,
+      "step": 200
+    },
+    {
+      "epoch": 0.00549151081072547,
+      "grad_norm": 0.4887239933013916,
+      "learning_rate": 5e-05,
+      "loss": 7.406,
+      "num_input_tokens_seen": 262144000,
+      "step": 250
+    },
+    {
+      "epoch": 0.006589812972870564,
+      "grad_norm": 0.7044657468795776,
+      "learning_rate": 6e-05,
+      "loss": 6.9708,
+      "num_input_tokens_seen": 314572800,
+      "step": 300
+    },
+    {
+      "epoch": 0.007688115135015658,
+      "grad_norm": 0.9420009255409241,
+      "learning_rate": 7.000000000000001e-05,
+      "loss": 6.6177,
+      "num_input_tokens_seen": 367001600,
+      "step": 350
+    },
+    {
+      "epoch": 0.008786417297160752,
+      "grad_norm": 1.4098254442214966,
+      "learning_rate": 8e-05,
+      "loss": 6.3285,
+      "num_input_tokens_seen": 419430400,
+      "step": 400
+    },
+    {
+      "epoch": 0.009884719459305845,
+      "grad_norm": 0.5596774220466614,
+      "learning_rate": 8.999999999999999e-05,
+      "loss": 6.0918,
+      "num_input_tokens_seen": 471859200,
+      "step": 450
+    },
+    {
+      "epoch": 0.01098302162145094,
+      "grad_norm": 0.5934723615646362,
+      "learning_rate": 0.0001,
+      "loss": 5.8926,
+      "num_input_tokens_seen": 524288000,
+      "step": 500
+    },
+    {
+      "epoch": 0.01098302162145094,
+      "eval_loss": 5.72648286819458,
+      "eval_runtime": 65.255,
+      "eval_samples_per_second": 76.622,
+      "eval_steps_per_second": 19.156,
+      "num_input_tokens_seen": 524288000,
+      "step": 500
+    },
+    {
+      "epoch": 0.012081323783596033,
+      "grad_norm": 0.6180132627487183,
+      "learning_rate": 0.00011,
+      "loss": 5.713,
+      "num_input_tokens_seen": 576716800,
+      "step": 550
+    },
+    {
+      "epoch": 0.013179625945741128,
+      "grad_norm": 0.7194430232048035,
+      "learning_rate": 0.00012,
+      "loss": 5.5604,
+      "num_input_tokens_seen": 629145600,
+      "step": 600
+    },
+    {
+      "epoch": 0.014277928107886221,
+      "grad_norm": 0.7763974070549011,
+      "learning_rate": 0.00013000000000000002,
+      "loss": 5.4212,
+      "num_input_tokens_seen": 681574400,
+      "step": 650
+    },
+    {
+      "epoch": 0.015376230270031316,
+      "grad_norm": 0.7948254942893982,
+      "learning_rate": 0.00014000000000000001,
+      "loss": 5.2875,
+      "num_input_tokens_seen": 734003200,
+      "step": 700
+    },
+    {
+      "epoch": 0.01647453243217641,
+      "grad_norm": 0.7185749411582947,
+      "learning_rate": 0.00015,
+      "loss": 5.1765,
+      "num_input_tokens_seen": 786432000,
+      "step": 750
+    },
+    {
+      "epoch": 0.017572834594321504,
+      "grad_norm": 0.673218846321106,
+      "learning_rate": 0.00016,
+      "loss": 5.0599,
+      "num_input_tokens_seen": 838860800,
+      "step": 800
+    },
+    {
+      "epoch": 0.018671136756466596,
+      "grad_norm": 0.6499584317207336,
+      "learning_rate": 0.00017,
+      "loss": 4.9475,
+      "num_input_tokens_seen": 891289600,
+      "step": 850
+    },
+    {
+      "epoch": 0.01976943891861169,
+      "grad_norm": 0.9044798016548157,
+      "learning_rate": 0.00017999999999999998,
+      "loss": 4.8334,
+      "num_input_tokens_seen": 943718400,
+      "step": 900
+    },
+    {
+      "epoch": 0.020867741080756785,
+      "grad_norm": 0.886431872844696,
+      "learning_rate": 0.00019,
+      "loss": 4.723,
+      "num_input_tokens_seen": 996147200,
+      "step": 950
+    },
+    {
+      "epoch": 0.02196604324290188,
+      "grad_norm": 0.6721145510673523,
+      "learning_rate": 0.0002,
+      "loss": 4.6106,
+      "num_input_tokens_seen": 1048576000,
+      "step": 1000
+    },
+    {
+      "epoch": 0.02196604324290188,
+      "eval_loss": 4.456684589385986,
+      "eval_runtime": 66.2606,
+      "eval_samples_per_second": 75.46,
+      "eval_steps_per_second": 18.865,
+      "num_input_tokens_seen": 1048576000,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 200000,
+  "num_input_tokens_seen": 1048576000,
+  "num_train_epochs": 5,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.97171715964928e+17,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b69959aab94b5eb1fc8797807cf131c6d7414b62cd03941326514e170b23af6
+size 6008