Finetune on MentalChat16K - eval_loss: 0.6693

README.md

@@ -19,7 +19,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [meta-llama/Llama-3.1-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.7052
+- Loss: 0.6542
 
 ## Model description
 

training_metrics.json

@@ -0,0 +1,46 @@
+{
+  "model": "llama71b-mental-health",
+  "base_model": "meta-llama/Llama-3.1-70B-Instruct",
+  "dataset": "ShenLab/MentalChat16K",
+  "lora_config": {
+    "rank": 64,
+    "alpha": 128,
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj",
+      "gate_proj",
+      "up_proj",
+      "down_proj"
+    ],
+    "dropout": 0.1
+  },
+  "training": {
+    "final_train_loss": 0.5772495049900479,
+    "total_steps": 1800,
+    "epochs": 3,
+    "learning_rate": 8e-05,
+    "per_device_batch_size": 1,
+    "gradient_accumulation": 8
+  },
+  "evaluation": {
+    "eval_loss": 0.6692664623260498,
+    "eval_runtime": 354.8904,
+    "eval_samples_per_second": 1.33,
+    "eval_steps_per_second": 1.33,
+    "epoch": 2.691228726388629
+  },
+  "test_eval": {
+    "eval_loss": 0.6542104482650757,
+    "eval_runtime": 355.8479,
+    "eval_samples_per_second": 1.326,
+    "eval_steps_per_second": 1.326,
+    "epoch": 2.691228726388629
+  },
+  "dataset_stats": {
+    "train_size": 5347,
+    "val_size": 472,
+    "test_size": 472
+  }
+}