End of training

README.md

@@ -30,6 +30,7 @@ debug: null
 deepspeed: null
 early_stopping_patience: null
 eval_max_new_tokens: 128
+eval_sample_packing: false
 eval_table_size: null
 evals_per_epoch: 4
 flash_attention: true
@@ -42,29 +43,29 @@ group_by_length: false
 hub_model_id: besimray/test
 hub_strategy: checkpoint
 hub_token: null
-learning_rate: 5.0e-05
+learning_rate: 0.0002
 load_in_4bit: false
 load_in_8bit: true
 local_rank: null
 logging_steps: 1
-lora_alpha: 64
+lora_alpha: 32
 lora_dropout: 0.05
 lora_fan_in_fan_out: null
 lora_model_dir: null
-lora_r: 24
+lora_r: 16
 lora_target_linear: true
 lr_scheduler: cosine
-max_steps: 10
-micro_batch_size: 7
+max_steps: 150
+micro_batch_size: 2
 mlflow_experiment_name: mhenrichsen/alpaca_2k_test
 model_type: LlamaForCausalLM
-num_epochs: 20
+num_epochs: 3
 optimizer: adamw_bnb_8bit
 output_dir: miner_id_besimray
 pad_to_sequence_len: false
 resume_from_checkpoint: null
 s2_attention: null
-sample_packing: false
+sample_packing: true
 save_steps: 5
 save_strategy: steps
 sequence_len: 4096
@@ -78,7 +79,7 @@ wandb_mode: online
 wandb_project: Public_TuningSN
 wandb_run: miner_id_24
 wandb_runid: 383a850e-bb15-45a2-8f4b-fc96eb001a74
-warmup_steps: 100
+warmup_steps: 10
 weight_decay: 0.0
 xformers_attention: null
 
@@ -90,7 +91,7 @@ xformers_attention: null
 
 This model is a fine-tuned version of [unsloth/Llama-3.2-1B-Instruct](https://huggingface.co/unsloth/Llama-3.2-1B-Instruct) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.2638
+- Loss: 1.1380
 
 ## Model description
 
@@ -109,31 +110,32 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-05
-- train_batch_size: 7
-- eval_batch_size: 7
+- learning_rate: 0.0002
+- train_batch_size: 2
+- eval_batch_size: 2
 - seed: 42
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 28
+- total_train_batch_size: 8
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
-- lr_scheduler_warmup_steps: 100
-- training_steps: 10
+- lr_scheduler_warmup_steps: 10
+- training_steps: 30
 
 ### Training results
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 1.3327        | 0.0147 | 1    | 1.2694          |
-| 1.1887        | 0.0294 | 2    | 1.2676          |
-| 1.5761        | 0.0441 | 3    | 1.2679          |
-| 1.3197        | 0.0588 | 4    | 1.2693          |
-| 1.3721        | 0.0735 | 5    | 1.2674          |
-| 1.4327        | 0.0882 | 6    | 1.2674          |
-| 1.2795        | 0.1029 | 7    | 1.2692          |
-| 1.4695        | 0.1176 | 8    | 1.2674          |
-| 1.4243        | 0.1324 | 9    | 1.2657          |
-| 1.4099        | 0.1471 | 10   | 1.2638          |
+| 1.3356        | 0.0952 | 1    | 1.2672          |
+| 1.2159        | 0.2857 | 3    | 1.2534          |
+| 1.2716        | 0.5714 | 6    | 1.2098          |
+| 1.2697        | 0.8571 | 9    | 1.1910          |
+| 1.2243        | 1.1190 | 12   | 1.1755          |
+| 1.1981        | 1.4048 | 15   | 1.1604          |
+| 1.1662        | 1.6905 | 18   | 1.1496          |
+| 1.1879        | 1.9762 | 21   | 1.1424          |
+| 1.198         | 2.2381 | 24   | 1.1413          |
+| 1.1981        | 2.5238 | 27   | 1.1411          |
+| 1.1377        | 2.8095 | 30   | 1.1380          |
 
 
 ### Framework versions

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:339f69e146da45ed44b91d2e55f3389d74d840b0727be524af0a85b4f71ec13e
-size 67713738
+oid sha256:267858af8019eb7e782699cae15101562b55ccdf93cf3dc7d1d602d0c507ea90
+size 45169354