End of training

README.md

@@ -66,11 +66,11 @@ lora_model_dir: null
 lora_r: 32
 lora_target_linear: true
 lr_scheduler: cosine
-max_steps: 100
+max_steps: 500
 micro_batch_size: 1
 mlflow_experiment_name: /tmp/0fe9be330b40f25d_train_data.json
 model_type: AutoModelForCausalLM
-num_epochs: 1
+num_epochs: 4
 optimizer: adamw_bnb_8bit
 output_dir: miner_id_24
 pad_to_sequence_len: true
@@ -103,7 +103,7 @@ xformers_attention: null
 
 This model is a fine-tuned version of [Qwen/Qwen2.5-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.9689
+- Loss: 1.0580
 
 ## Model description
 
@@ -131,13 +131,17 @@ The following hyperparameters were used during training:
 - optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 10
-- training_steps: 100
+- training_steps: 500
 
 ### Training results
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 1.6039        | 0.0113 | 100  | 1.9689          |
+| 5.4801        | 0.0001 | 1    | 6.2892          |
+| 1.4237        | 0.0142 | 125  | 1.3264          |
+| 0.953         | 0.0284 | 250  | 1.1498          |
+| 0.8279        | 0.0425 | 375  | 1.0727          |
+| 1.0675        | 0.0567 | 500  | 1.0580          |
 
 
 ### Framework versions

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
-    "q_proj",
-    "v_proj",
     "o_proj",
     "k_proj",
+    "down_proj",
     "up_proj",
-    "down_proj"
+    "gate_proj",
+    "q_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d7fe1df43ac934e7263717d724f4ce2d3389c91b06e7f11e0b38e8f1f4d4209
+oid sha256:f3784fa22e586a753a5a30e8dd78c3a7f19b80e00299e7e20aa257775efd588c
 size 70506570

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:82281a0105057b4df432900a373bdbb0234a05bf0b3bc9f1f5106eb71273794a
+oid sha256:e78bbe4c88e10e5a49e4b7812347a0bc571d773dac6b5888ea1bd69c52f84b16
 size 70430032

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e3b00705b00812815e695fb883fc4148f6c255aedbce9f8d586fee3c83d859e
+oid sha256:03faafebfe7e7e6a881ee31a6d1c8c276772d4b35b0877e4552974cf0e391a70
 size 6776