End of training

README.md

@@ -1,6 +1,5 @@
 ---
 library_name: peft
-license: llama3.2
 base_model: unsloth/Llama-3.2-1B-Instruct
 tags:
 - axolotl
@@ -24,23 +23,14 @@ bf16: auto
 chat_template: llama3
 dataset_prepared_path: null
 datasets:
-- data_files:
-  - MATH-Hard_train_data.json
-  ds_type: json
-  path: /workspace/input_data/MATH-Hard_train_data.json
-  type:
-    field_input: problem
-    field_instruction: solution
-    field_output: type
-    system_format: '{system}'
-    system_prompt: ''
+- path: mhenrichsen/alpaca_2k_test
+  type: alpaca
 debug: null
 deepspeed: null
-early_stopping_patience: 3
+early_stopping_patience: null
 eval_max_new_tokens: 128
-eval_sample_packing: false
-eval_steps: 20
 eval_table_size: null
+evals_per_epoch: 4
 flash_attention: true
 fp16: null
 fsdp: null
@@ -63,18 +53,18 @@ lora_model_dir: null
 lora_r: 16
 lora_target_linear: true
 lr_scheduler: cosine
-max_steps: 150
-micro_batch_size: 10
-mlflow_experiment_name: /tmp/MATH-Hard_train_data.json
+max_steps: 10
+micro_batch_size: 2
+mlflow_experiment_name: mhenrichsen/alpaca_2k_test
 model_type: LlamaForCausalLM
-num_epochs: 10
+num_epochs: 1
 optimizer: adamw_bnb_8bit
-output_dir: miner_id_besimray
-pad_to_sequence_len: false
+output_dir: miner_id_24
+pad_to_sequence_len: true
 resume_from_checkpoint: null
 s2_attention: null
 sample_packing: false
-save_steps: 20
+save_steps: 5
 save_strategy: steps
 sequence_len: 4096
 strict: false
@@ -86,9 +76,9 @@ wandb_entity: besimray24-rayon
 wandb_mode: online
 wandb_project: Public_TuningSN
 wandb_run: miner_id_24
-wandb_runid: 3882ca50-f5d8-4a62-83cf-33e7720e8c52
+wandb_runid: 383a850e-bb15-45a2-8f4b-fc96eb001a74
 warmup_steps: 10
-weight_decay: 0.01
+weight_decay: 0.0
 xformers_attention: null
 
 ```
@@ -99,7 +89,7 @@ xformers_attention: null
 
 This model is a fine-tuned version of [unsloth/Llama-3.2-1B-Instruct](https://huggingface.co/unsloth/Llama-3.2-1B-Instruct) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0766
+- Loss: 1.2201
 
 ## Model description
 
@@ -119,28 +109,24 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0002
-- train_batch_size: 10
-- eval_batch_size: 10
+- train_batch_size: 2
+- eval_batch_size: 2
 - seed: 42
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 40
+- total_train_batch_size: 8
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 10
-- training_steps: 150
+- training_steps: 10
 
 ### Training results
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 9.1           | 0.0129 | 1    | 8.9962          |
-| 0.3746        | 0.2572 | 20   | 0.3471          |
-| 0.2618        | 0.5145 | 40   | 0.1247          |
-| 0.106         | 0.7717 | 60   | 0.1141          |
-| 0.1457        | 1.0289 | 80   | 0.1035          |
-| 0.0493        | 1.2862 | 100  | 0.0947          |
-| 0.1237        | 1.5434 | 120  | 0.0765          |
-| 0.0294        | 1.8006 | 140  | 0.0766          |
+| 1.3218        | 0.0042 | 1    | 1.2625          |
+| 1.3092        | 0.0126 | 3    | 1.2572          |
+| 1.4991        | 0.0253 | 6    | 1.2118          |
+| 1.2957        | 0.0379 | 9    | 1.2201          |
 
 
 ### Framework versions

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9e5432c0904d2ebf3acbe3d9dc2636391218bc21433147b98fca1a4e966c12f
+oid sha256:675843a59309216350febea0140e8a214bf010da20f9877d322a7fda208f3c3d
 size 45169354