Training in progress, step 400

axolotl_config.yaml

@@ -13,7 +13,9 @@ datasets:
     no_input_format: '{instruction}'
     system_format: '{system}'
     system_prompt: ''
-eval_steps: 20
+device_map: auto
+eval_sample_packing: false
+eval_steps: 200
 flash_attention: true
 gpu_memory_limit: 80GiB
 group_by_length: true
@@ -23,17 +25,19 @@ learning_rate: 0.0002
 logging_steps: 10
 lr_scheduler: cosine
 max_steps: 2500
-micro_batch_size: 2
+micro_batch_size: 4
 model_type: AutoModelForCausalLM
 num_epochs: 100
 optimizer: adamw_bnb_8bit
 output_dir: /root/.sn56/axolotl/outputs/SmolLM-360M-Instruct
 pad_to_sequence_len: true
 resize_token_embeddings_to_32x: false
-save_steps: 40
+sample_packing: false
+save_steps: 400
 save_total_limit: 1
 sequence_len: 2048
 tokenizer_type: GPT2TokenizerFast
+torch_dtype: bf16
 trust_remote_code: true
 val_set_size: 0.1
 wandb_entity: ''

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89da288ea916bc87e2dc309ae736f2a5f22442c411406161ba1505c8cd18f6fd
+oid sha256:8c3b20dc5667964ffe2baf981e662e4a77f20dd3ac7eecffcec9fa4d6eb9d179
 size 723674912

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff2d010d71ffe14af88bc9959221d3983bd2a3f394734b7d7b835d485397643f
-size 6712
+oid sha256:8728015aed6a4d50cfee4eda0cc3ff69e5a4901af146b3fd10ebc2972270b3a8
+size 6840