End of training

README.md

@@ -25,8 +25,8 @@ is_llama_derived_model: true
 
 hub_model_id: noeloco/camel-lora
 
-load_in_8bit: false
-load_in_4bit: true
+load_in_8bit: true
+load_in_4bit: false
 strict: false
 
 datasets:
@@ -44,13 +44,21 @@ sequence_len: 4096
 sample_packing: false
 pad_to_sequence_len: true
 
-adapter: qlora
+adapter: lora
 lora_model_dir:
-lora_r: 32
+lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
-lora_target_linear: true
+lora_target_linear: false
 lora_fan_in_fan_out:
+lora_target_modules:
+  - q_proj
+  - v_proj
+  - k_proj
+  - o_proj
+  - gate_proj
+  - down_proj
+  - up_proj
 
 wandb_project: runpod1
 wandb_entity:
@@ -98,9 +106,9 @@ special_tokens:
 
 # camel-lora
 
-This model is a fine-tuned version of [codellama/CodeLlama-7b-hf](https://huggingface.co/codellama/CodeLlama-7b-hf) on the None dataset.
+This model is a fine-tuned version of [codellama/CodeLlama-7b-hf](https://huggingface.co/codellama/CodeLlama-7b-hf) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0290
+- Loss: 0.0294
 
 ## Model description
 
@@ -134,20 +142,20 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 1.7685        | 0.06  | 1    | 2.5524          |
-| 1.8762        | 0.29  | 5    | 2.4927          |
-| 1.215         | 0.57  | 10   | 1.4546          |
-| 0.484         | 0.86  | 15   | 0.7250          |
-| 0.3667        | 1.14  | 20   | 0.4146          |
-| 0.1638        | 1.43  | 25   | 0.2123          |
-| 0.2948        | 1.71  | 30   | 0.0980          |
-| 0.2003        | 2.0   | 35   | 0.0629          |
-| 0.0888        | 2.29  | 40   | 0.0577          |
-| 0.0918        | 2.57  | 45   | 0.0414          |
-| 0.0931        | 2.86  | 50   | 0.0363          |
-| 0.0982        | 3.14  | 55   | 0.0304          |
-| 0.0849        | 3.43  | 60   | 0.0289          |
-| 0.0511        | 3.71  | 65   | 0.0290          |
+| 1.7211        | 0.06  | 1    | 2.5058          |
+| 1.834         | 0.29  | 5    | 2.4238          |
+| 1.1688        | 0.57  | 10   | 1.3647          |
+| 0.483         | 0.86  | 15   | 0.7108          |
+| 0.3742        | 1.14  | 20   | 0.3942          |
+| 0.1581        | 1.43  | 25   | 0.2196          |
+| 0.2905        | 1.71  | 30   | 0.0822          |
+| 0.1803        | 2.0   | 35   | 0.0548          |
+| 0.0799        | 2.29  | 40   | 0.0543          |
+| 0.0932        | 2.57  | 45   | 0.0390          |
+| 0.0851        | 2.86  | 50   | 0.0328          |
+| 0.096         | 3.14  | 55   | 0.0287          |
+| 0.086         | 3.43  | 60   | 0.0289          |
+| 0.0459        | 3.71  | 65   | 0.0294          |
 
 
 ### Framework versions

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:430885f7d5e76c656bb16d0c30097c03805f898ac3586031af9f6c6c1d88520a
-size 160069834
+oid sha256:076ff87239662d4acee0599b11ba12bf120bc15669aa1bc54da375fe3d51e040
+size 80115210