End of training

README.md

@@ -26,7 +26,7 @@ is_llama_derived_model: true
 hub_model_id: noeloco/camel-lora
 
 load_in_8bit: false
-load_in_4bit: false
+load_in_4bit: true
 strict: false
 
 datasets:
@@ -40,14 +40,14 @@ val_set_size: 0.05
 output_dir: ./lora-out
 chat_template: chatml
 
-sequence_len: 2048
+sequence_len: 4096
 sample_packing: false
 pad_to_sequence_len: true
 
-adapter: lora
+adapter: qlora
 lora_model_dir:
-lora_r: 16
-lora_alpha: 8
+lora_r: 32
+lora_alpha: 16
 lora_dropout: 0.05
 lora_target_linear: true
 lora_fan_in_fan_out:
@@ -67,9 +67,9 @@ learning_rate: 0.0002
 
 train_on_inputs: false
 group_by_length: false
-bf16: true
+bf16: auto
 fp16: false
-tf32: false
+tf32: true
 
 gradient_checkpointing: true
 early_stopping_patience:
@@ -100,7 +100,7 @@ special_tokens:
 
 This model is a fine-tuned version of [codellama/CodeLlama-7b-hf](https://huggingface.co/codellama/CodeLlama-7b-hf) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0383
+- Loss: 0.0290
 
 ## Model description
 
@@ -134,20 +134,20 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 1.7285        | 0.06  | 1    | 2.5189          |
-| 1.8487        | 0.29  | 5    | 2.4980          |
-| 1.4443        | 0.57  | 10   | 1.9379          |
-| 0.7471        | 0.86  | 15   | 1.0148          |
-| 0.561         | 1.14  | 20   | 0.5721          |
-| 0.2245        | 1.43  | 25   | 0.3640          |
-| 0.3456        | 1.71  | 30   | 0.1683          |
-| 0.2138        | 2.0   | 35   | 0.1051          |
-| 0.1145        | 2.29  | 40   | 0.0834          |
-| 0.1193        | 2.57  | 45   | 0.0526          |
-| 0.1083        | 2.86  | 50   | 0.0436          |
-| 0.1388        | 3.14  | 55   | 0.0387          |
-| 0.1102        | 3.43  | 60   | 0.0385          |
-| 0.0628        | 3.71  | 65   | 0.0383          |
+| 1.7685        | 0.06  | 1    | 2.5524          |
+| 1.8762        | 0.29  | 5    | 2.4927          |
+| 1.215         | 0.57  | 10   | 1.4546          |
+| 0.484         | 0.86  | 15   | 0.7250          |
+| 0.3667        | 1.14  | 20   | 0.4146          |
+| 0.1638        | 1.43  | 25   | 0.2123          |
+| 0.2948        | 1.71  | 30   | 0.0980          |
+| 0.2003        | 2.0   | 35   | 0.0629          |
+| 0.0888        | 2.29  | 40   | 0.0577          |
+| 0.0918        | 2.57  | 45   | 0.0414          |
+| 0.0931        | 2.86  | 50   | 0.0363          |
+| 0.0982        | 3.14  | 55   | 0.0304          |
+| 0.0849        | 3.43  | 60   | 0.0289          |
+| 0.0511        | 3.71  | 65   | 0.0290          |
 
 
 ### Framework versions

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b66e19266959041f6863e2ae34bc5aee2db2a88b45bac11641d5327b69dc9df
-size 80115914
+oid sha256:430885f7d5e76c656bb16d0c30097c03805f898ac3586031af9f6c6c1d88520a
+size 160069834