Finetune on MentalChat16K - eval_loss: 0.7816

README.md

@@ -16,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.8010
+- Loss: 0.7700
 
 ## Model description
 

training_metrics.json

@@ -0,0 +1,46 @@
+{
+  "model": "tinyllama-mental-health",
+  "base_model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "dataset": "ShenLab/MentalChat16K",
+  "lora_config": {
+    "rank": 16,
+    "alpha": 32,
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj",
+      "gate_proj",
+      "up_proj",
+      "down_proj"
+    ],
+    "dropout": 0.1
+  },
+  "training": {
+    "final_train_loss": 0.7040511888504029,
+    "total_steps": 2500,
+    "epochs": 4,
+    "learning_rate": 0.0002,
+    "per_device_batch_size": 4,
+    "gradient_accumulation": 2
+  },
+  "evaluation": {
+    "eval_loss": 0.7816067934036255,
+    "eval_runtime": 18.522,
+    "eval_samples_per_second": 25.483,
+    "eval_steps_per_second": 6.371,
+    "epoch": 3.7397157816005984
+  },
+  "test_eval": {
+    "eval_loss": 0.7700048089027405,
+    "eval_runtime": 18.7703,
+    "eval_samples_per_second": 25.146,
+    "eval_steps_per_second": 6.287,
+    "epoch": 3.7397157816005984
+  },
+  "dataset_stats": {
+    "train_size": 5347,
+    "val_size": 472,
+    "test_size": 472
+  }
+}