Model save

README.md

@@ -1,36 +1,36 @@
 ---
-license: other
+license: gemma
 library_name: peft
 tags:
-- llama-factory
-- lora
 - trl
 - dpo
+- llama-factory
+- lora
 - generated_from_trainer
 base_model: google/gemma-7b-it
 model-index:
-- name: orpo
+- name: Gemma-7B-It-ORPO
   results: []
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# orpo
+# Gemma-7B-It-ORPO
 
-This model is a fine-tuned version of [google/gemma-7b-it](https://huggingface.co/google/gemma-7b-it) on the dpo_mix_en dataset.
+This model is a fine-tuned version of [google/gemma-7b-it](https://huggingface.co/google/gemma-7b-it) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 4.4269
-- Rewards/chosen: -0.4347
-- Rewards/rejected: -0.4158
-- Rewards/accuracies: 0.0
-- Rewards/margins: -0.0190
-- Logps/rejected: -4.1575
-- Logps/chosen: -4.3475
-- Logits/rejected: 198.3044
-- Logits/chosen: 195.7259
-- Sft Loss: 4.3475
-- Odds Ratio Loss: 0.7941
+- Loss: 1.3471
+- Rewards/chosen: -0.1281
+- Rewards/rejected: -0.1500
+- Rewards/accuracies: 0.5610
+- Rewards/margins: 0.0219
+- Logps/rejected: -1.5004
+- Logps/chosen: -1.2814
+- Logits/rejected: 254.6614
+- Logits/chosen: 254.4679
+- Sft Loss: 1.2814
+- Odds Ratio Loss: 0.6571
 
 ## Model description
 
@@ -58,10 +58,15 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 0.1
-- num_epochs: 1.0
+- num_epochs: 3.0
 
 ### Training results
 
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Sft Loss | Odds Ratio Loss |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------:|:---------------:|
+| 1.5041        | 0.8891 | 500  | 1.4185          | -0.1352        | -0.1564          | 0.5530             | 0.0212          | -1.5644        | -1.3522      | 250.7549        | 250.6463      | 1.3522   | 0.6626          |
+| 1.428         | 1.7782 | 1000 | 1.3595          | -0.1294        | -0.1509          | 0.5600             | 0.0215          | -1.5091        | -1.2937      | 254.1350        | 253.9581      | 1.2937   | 0.6586          |
+| 1.3302        | 2.6673 | 1500 | 1.3471          | -0.1281        | -0.1500          | 0.5610             | 0.0219          | -1.5004        | -1.2814      | 254.6614        | 254.4679      | 1.2814   | 0.6571          |
 
 
 ### Framework versions

trainer_log.jsonl

@@ -151,3 +151,22 @@
 {"current_steps": 1490, "total_steps": 1686, "loss": 1.289, "accuracy": 0.59375, "learning_rate": 1.6490167940538343e-07, "epoch": 2.6494776617026004, "percentage": 88.37, "elapsed_time": "5:34:21", "remaining_time": "0:43:58"}
 {"current_steps": 1500, "total_steps": 1686, "loss": 1.3302, "accuracy": 0.512499988079071, "learning_rate": 1.4866882516191339e-07, "epoch": 2.6672593909757722, "percentage": 88.97, "elapsed_time": "5:36:38", "remaining_time": "0:41:44"}
 {"current_steps": 1500, "total_steps": 1686, "eval_loss": 1.347064733505249, "epoch": 2.6672593909757722, "percentage": 88.97, "elapsed_time": "5:40:15", "remaining_time": "0:42:11"}
+{"current_steps": 1510, "total_steps": 1686, "loss": 1.3335, "accuracy": 0.606249988079071, "learning_rate": 1.3325243551706057e-07, "epoch": 2.685041120248944, "percentage": 89.56, "elapsed_time": "5:42:29", "remaining_time": "0:39:55"}
+{"current_steps": 1520, "total_steps": 1686, "loss": 1.3537, "accuracy": 0.59375, "learning_rate": 1.1865786358165737e-07, "epoch": 2.702822849522116, "percentage": 90.15, "elapsed_time": "5:44:37", "remaining_time": "0:37:38"}
+{"current_steps": 1530, "total_steps": 1686, "loss": 1.4013, "accuracy": 0.581250011920929, "learning_rate": 1.0489017710262311e-07, "epoch": 2.720604578795288, "percentage": 90.75, "elapsed_time": "5:46:51", "remaining_time": "0:35:21"}
+{"current_steps": 1540, "total_steps": 1686, "loss": 1.3524, "accuracy": 0.53125, "learning_rate": 9.195415670326446e-08, "epoch": 2.73838630806846, "percentage": 91.34, "elapsed_time": "5:49:06", "remaining_time": "0:33:05"}
+{"current_steps": 1550, "total_steps": 1686, "loss": 1.3267, "accuracy": 0.518750011920929, "learning_rate": 7.985429422327384e-08, "epoch": 2.7561680373416317, "percentage": 91.93, "elapsed_time": "5:51:13", "remaining_time": "0:30:49"}
+{"current_steps": 1560, "total_steps": 1686, "loss": 1.425, "accuracy": 0.5562499761581421, "learning_rate": 6.859479115900818e-08, "epoch": 2.773949766614803, "percentage": 92.53, "elapsed_time": "5:53:23", "remaining_time": "0:28:32"}
+{"current_steps": 1570, "total_steps": 1686, "loss": 1.262, "accuracy": 0.5562499761581421, "learning_rate": 5.817955720457902e-08, "epoch": 2.791731495887975, "percentage": 93.12, "elapsed_time": "5:55:33", "remaining_time": "0:26:16"}
+{"current_steps": 1580, "total_steps": 1686, "loss": 1.3469, "accuracy": 0.53125, "learning_rate": 4.861220889427199e-08, "epoch": 2.809513225161147, "percentage": 93.71, "elapsed_time": "5:57:36", "remaining_time": "0:23:59"}
+{"current_steps": 1590, "total_steps": 1686, "loss": 1.3588, "accuracy": 0.612500011920929, "learning_rate": 3.9896068346758074e-08, "epoch": 2.827294954434319, "percentage": 94.31, "elapsed_time": "5:59:45", "remaining_time": "0:21:43"}
+{"current_steps": 1600, "total_steps": 1686, "loss": 1.3344, "accuracy": 0.5, "learning_rate": 3.203416211153832e-08, "epoch": 2.8450766837074903, "percentage": 94.9, "elapsed_time": "6:01:55", "remaining_time": "0:19:27"}
+{"current_steps": 1610, "total_steps": 1686, "loss": 1.3818, "accuracy": 0.543749988079071, "learning_rate": 2.5029220118019393e-08, "epoch": 2.8628584129806622, "percentage": 95.49, "elapsed_time": "6:04:11", "remaining_time": "0:17:11"}
+{"current_steps": 1620, "total_steps": 1686, "loss": 1.2541, "accuracy": 0.6312500238418579, "learning_rate": 1.8883674727586122e-08, "epoch": 2.880640142253834, "percentage": 96.09, "elapsed_time": "6:06:17", "remaining_time": "0:14:55"}
+{"current_steps": 1630, "total_steps": 1686, "loss": 1.3872, "accuracy": 0.4749999940395355, "learning_rate": 1.3599659889000639e-08, "epoch": 2.898421871527006, "percentage": 96.68, "elapsed_time": "6:08:32", "remaining_time": "0:12:39"}
+{"current_steps": 1640, "total_steps": 1686, "loss": 1.3386, "accuracy": 0.5687500238418579, "learning_rate": 9.179010397421528e-09, "epoch": 2.916203600800178, "percentage": 97.27, "elapsed_time": "6:10:56", "remaining_time": "0:10:24"}
+{"current_steps": 1650, "total_steps": 1686, "loss": 1.2157, "accuracy": 0.550000011920929, "learning_rate": 5.623261257296509e-09, "epoch": 2.93398533007335, "percentage": 97.86, "elapsed_time": "6:13:07", "remaining_time": "0:08:08"}
+{"current_steps": 1660, "total_steps": 1686, "loss": 1.3705, "accuracy": 0.5562499761581421, "learning_rate": 2.933647149357122e-09, "epoch": 2.9517670593465217, "percentage": 98.46, "elapsed_time": "6:15:16", "remaining_time": "0:05:52"}
+{"current_steps": 1670, "total_steps": 1686, "loss": 1.3264, "accuracy": 0.48124998807907104, "learning_rate": 1.1111020018930717e-09, "epoch": 2.969548788619693, "percentage": 99.05, "elapsed_time": "6:17:28", "remaining_time": "0:03:36"}
+{"current_steps": 1680, "total_steps": 1686, "loss": 1.2721, "accuracy": 0.6312500238418579, "learning_rate": 1.5625866646051813e-10, "epoch": 2.987330517892865, "percentage": 99.64, "elapsed_time": "6:19:37", "remaining_time": "0:01:21"}
+{"current_steps": 1686, "total_steps": 1686, "epoch": 2.997999555456768, "percentage": 100.0, "elapsed_time": "6:21:02", "remaining_time": "0:00:00"}