Training in progress, step 1000, checkpoint

last-checkpoint/config.json

@@ -0,0 +1,52 @@
+{
+  "architectures": [
+    "SmalLmForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.1,
+  "auto_map": {
+    "AutoConfig": "Azrail/smallm_70--config.SmalLmConfig",
+    "AutoModelForCausalLM": "Azrail/smallm_70--model.SmalLmForCausalLM"
+  },
+  "balancing_coef": 0.0001,
+  "bos_token_id": 1,
+  "embedding_dropout": 0.0,
+  "eos_token_id": 0,
+  "expert_size": 576,
+  "gate_noise": false,
+  "head_size": 64,
+  "hidden_size": 512,
+  "high_rotations": 32,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "layer_dropout": 0.1,
+  "low_rotations": 1,
+  "max_seq_len": 2048,
+  "mlp_bias": false,
+  "model_type": "smallm",
+  "moe_bias": false,
+  "moe_period": 2,
+  "no_moe_layers": 10,
+  "noisy_experts": false,
+  "num_attention_heads": 8,
+  "num_hidden_layers": 20,
+  "num_kv_heads": 2,
+  "original_seq_len": 1024,
+  "pad_token_id": 0,
+  "positional_bias_type": "rope",
+  "rms_affine": false,
+  "rms_norm_eps": 1e-06,
+  "rope_base": 100000,
+  "routed_experts": 8,
+  "shared_experts": 1,
+  "sliding_window_attention": true,
+  "sliding_window_context": 1024,
+  "sliding_window_period": 4,
+  "static_residual": false,
+  "token_experts": 3,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.50.3",
+  "use_cache": true,
+  "use_moe": false,
+  "vocab_size": 60000
+}

last-checkpoint/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "do_sample": true,
+  "max_new_tokens": 100,
+  "repetition_penalty": 1.2,
+  "top_p": 0.8,
+  "transformers_version": "4.50.3"
+}

last-checkpoint/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f3c22936a0095ac18daff8e6dcf0e5a3c013ec81937d069c3fc7ec2ce742258
+size 150625560

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ede7891a612e5bdf9d9f3ea5ba8ce2d972ac251ac865d5a7ca14b4f136bc03db
+size 602335994

last-checkpoint/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:521f432674dde45244a4689a033d9db107314cdff516b6bf8384096ef5f6c190
+size 14244

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1c35f90edf203b2f0483c8d4d666d73bb8fd60a06a785c4abd30a947f264d74
+size 1064

last-checkpoint/special_tokens_map.json

@@ -0,0 +1,27 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|beginoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

last-checkpoint/tokenizer_config.json

@@ -0,0 +1,120 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|beginoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<|reserved_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<|reserved_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<|reserved_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<|reserved_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<|reserved_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<|reserved_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<|reserved_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<|reserved_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<|reserved_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|beginoftext|>",
+  "chat_template": "{% for message in messages %}{% if message.get('role') is not none %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% else %}{{message['content'] + '<|im_end|>' + '\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}

last-checkpoint/trainer_state.json

@@ -0,0 +1,252 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.24140740517215364,
+  "eval_steps": 500,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.012070370258607683,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.5087507543753772e-06,
+      "loss": 2.681,
+      "mean_token_accuracy": 0.5007504414767027,
+      "num_input_tokens_seen": 25446880,
+      "num_tokens": 10166805.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.024140740517215365,
+      "grad_norm": 1.4921875,
+      "learning_rate": 3.0175015087507544e-06,
+      "loss": 2.6838,
+      "mean_token_accuracy": 0.500230128839612,
+      "num_input_tokens_seen": 51020960,
+      "num_tokens": 20365412.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.03621111077582305,
+      "grad_norm": 1.6484375,
+      "learning_rate": 4.526252263126131e-06,
+      "loss": 2.6811,
+      "mean_token_accuracy": 0.5007199160754681,
+      "num_input_tokens_seen": 76507776,
+      "num_tokens": 30424121.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.04828148103443073,
+      "grad_norm": 1.4765625,
+      "learning_rate": 6.035003017501509e-06,
+      "loss": 2.6797,
+      "mean_token_accuracy": 0.500353229418397,
+      "num_input_tokens_seen": 101977920,
+      "num_tokens": 40650709.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.06035185129303841,
+      "grad_norm": 1.421875,
+      "learning_rate": 7.543753771876886e-06,
+      "loss": 2.6464,
+      "mean_token_accuracy": 0.5040991773456335,
+      "num_input_tokens_seen": 127536896,
+      "num_tokens": 50830609.0,
+      "step": 250
+    },
+    {
+      "epoch": 0.0724222215516461,
+      "grad_norm": 1.140625,
+      "learning_rate": 9.052504526252262e-06,
+      "loss": 2.654,
+      "mean_token_accuracy": 0.5034311909228564,
+      "num_input_tokens_seen": 153048544,
+      "num_tokens": 60898431.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.08449259181025377,
+      "grad_norm": 1.0234375,
+      "learning_rate": 1.0561255280627642e-05,
+      "loss": 2.625,
+      "mean_token_accuracy": 0.5059863431751728,
+      "num_input_tokens_seen": 178595392,
+      "num_tokens": 71195944.0,
+      "step": 350
+    },
+    {
+      "epoch": 0.09656296206886146,
+      "grad_norm": 0.91015625,
+      "learning_rate": 1.2070006035003018e-05,
+      "loss": 2.6079,
+      "mean_token_accuracy": 0.5077864757180214,
+      "num_input_tokens_seen": 204085184,
+      "num_tokens": 81381172.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.10863333232746915,
+      "grad_norm": 0.78515625,
+      "learning_rate": 1.3578756789378394e-05,
+      "loss": 2.6083,
+      "mean_token_accuracy": 0.5078243865072727,
+      "num_input_tokens_seen": 229609824,
+      "num_tokens": 91569720.0,
+      "step": 450
+    },
+    {
+      "epoch": 0.12070370258607682,
+      "grad_norm": 0.734375,
+      "learning_rate": 1.5087507543753773e-05,
+      "loss": 2.591,
+      "num_input_tokens_seen": 255055520,
+      "step": 500
+    },
+    {
+      "epoch": 0.12070370258607682,
+      "eval_loss": 2.492645263671875,
+      "eval_mean_token_accuracy": 0.5286665781386491,
+      "eval_num_tokens": 101801812.0,
+      "eval_runtime": 126.4646,
+      "eval_samples_per_second": 84.704,
+      "eval_steps_per_second": 21.176,
+      "num_input_tokens_seen": 255055520,
+      "step": 500
+    },
+    {
+      "epoch": 0.1327740728446845,
+      "grad_norm": 0.70703125,
+      "learning_rate": 1.659625829812915e-05,
+      "loss": 2.5826,
+      "mean_token_accuracy": 0.5089771485328675,
+      "num_input_tokens_seen": 280435680,
+      "num_tokens": 111840270.0,
+      "step": 550
+    },
+    {
+      "epoch": 0.1448444431032922,
+      "grad_norm": 0.65625,
+      "learning_rate": 1.8105009052504525e-05,
+      "loss": 2.5689,
+      "mean_token_accuracy": 0.5112711164355278,
+      "num_input_tokens_seen": 305931936,
+      "num_tokens": 121987982.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.15691481336189989,
+      "grad_norm": 0.75,
+      "learning_rate": 1.9613759806879906e-05,
+      "loss": 2.55,
+      "mean_token_accuracy": 0.5135074812173843,
+      "num_input_tokens_seen": 331404160,
+      "num_tokens": 132144416.0,
+      "step": 650
+    },
+    {
+      "epoch": 0.16898518362050755,
+      "grad_norm": 0.58984375,
+      "learning_rate": 2.1122510561255283e-05,
+      "loss": 2.5497,
+      "mean_token_accuracy": 0.513639942780137,
+      "num_input_tokens_seen": 356956832,
+      "num_tokens": 142378323.0,
+      "step": 700
+    },
+    {
+      "epoch": 0.18105555387911523,
+      "grad_norm": 0.486328125,
+      "learning_rate": 2.2631261315630658e-05,
+      "loss": 2.5395,
+      "mean_token_accuracy": 0.5144028599560261,
+      "num_input_tokens_seen": 382529376,
+      "num_tokens": 152623392.0,
+      "step": 750
+    },
+    {
+      "epoch": 0.19312592413772292,
+      "grad_norm": 0.5234375,
+      "learning_rate": 2.4140012070006035e-05,
+      "loss": 2.5309,
+      "mean_token_accuracy": 0.5156180351227522,
+      "num_input_tokens_seen": 408021056,
+      "num_tokens": 162827766.0,
+      "step": 800
+    },
+    {
+      "epoch": 0.2051962943963306,
+      "grad_norm": 0.447265625,
+      "learning_rate": 2.5648762824381413e-05,
+      "loss": 2.5241,
+      "mean_token_accuracy": 0.5165234027802944,
+      "num_input_tokens_seen": 433622464,
+      "num_tokens": 172979899.0,
+      "step": 850
+    },
+    {
+      "epoch": 0.2172666646549383,
+      "grad_norm": 0.55078125,
+      "learning_rate": 2.7157513578756787e-05,
+      "loss": 2.5219,
+      "mean_token_accuracy": 0.5160751658678054,
+      "num_input_tokens_seen": 459074080,
+      "num_tokens": 183138615.0,
+      "step": 900
+    },
+    {
+      "epoch": 0.22933703491354598,
+      "grad_norm": 0.482421875,
+      "learning_rate": 2.866626433313217e-05,
+      "loss": 2.511,
+      "mean_token_accuracy": 0.5176014001667499,
+      "num_input_tokens_seen": 484598624,
+      "num_tokens": 193253896.0,
+      "step": 950
+    },
+    {
+      "epoch": 0.24140740517215364,
+      "grad_norm": 0.6015625,
+      "learning_rate": 3.0175015087507546e-05,
+      "loss": 2.4857,
+      "num_input_tokens_seen": 510077440,
+      "step": 1000
+    },
+    {
+      "epoch": 0.24140740517215364,
+      "eval_loss": 2.4022138118743896,
+      "eval_mean_token_accuracy": 0.5380484161312853,
+      "eval_num_tokens": 203431197.0,
+      "eval_runtime": 126.7655,
+      "eval_samples_per_second": 84.503,
+      "eval_steps_per_second": 21.126,
+      "num_input_tokens_seen": 510077440,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 8284,
+  "num_input_tokens_seen": 510077440,
+  "num_train_epochs": 2,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.364506935558144e+17,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02b13fc8d7c893a956ab01c588fd523c765a348a70de0153e06e4429d50ed8ee
+size 5752