End of training

README.md

@@ -5,7 +5,15 @@ tags:
 - axolotl
 - generated_from_trainer
 datasets:
+- lemon-mint/Korean-FineTome-100k
 - lemon-mint/smol-koreantalk
+- heegyu/open-korean-instructions-v20231020
+- trillionlabs/multisystem-curated
+- allenai/tulu-3-sft-personas-instruction-following
+- coastral/korean-writing-style-instruct
+- devngho/korean-instruction-mix
+- youjunhyeok/Magpie-Pro-300K-Filtered-ko
+- youjunhyeok/smoltalk-ko-translate
 model-index:
 - name: tiny-ko-124m-sft
   results: []
@@ -33,6 +41,14 @@ strict: false
 
 chat_template: chatml
 datasets:
+  - path: lemon-mint/Korean-FineTome-100k
+    type: chat_template
+    split: train
+    field_messages: messages
+    message_property_mappings:
+      role: role
+      content: content
+
   - path: lemon-mint/smol-koreantalk
     type: chat_template
     split: train
@@ -41,6 +57,64 @@ datasets:
       role: role
       content: content
 
+  - path: heegyu/open-korean-instructions-v20231020
+    type: chat_template
+    split: train
+    field_messages: conversations
+    message_property_mappings:
+      role: from
+      content: value
+    roles:
+      user: ["human", "user"]
+      assistant: ["gpt", "assistant", "bot"]
+      system: ["system", "input"]
+  - path: trillionlabs/multisystem-curated
+    type: chat_template
+    split: train
+    field_messages: messages
+    message_property_mappings:
+      role: role
+      content: content
+  - path: allenai/tulu-3-sft-personas-instruction-following
+    type: chat_template
+    split: train
+    field_messages: messages
+    message_property_mappings:
+      role: role
+      content: content
+  - path: coastral/korean-writing-style-instruct
+    type: chat_template
+    split: train
+    field_messages: conversations
+    message_property_mappings:
+      role: from
+      content: value
+
+  - path: devngho/korean-instruction-mix
+    type: chat_template
+    split: train
+    field_messages: messages
+    message_property_mappings:
+      role: from
+      content: value
+
+  - path: youjunhyeok/Magpie-Pro-300K-Filtered-ko
+    type: chat_template
+    split: train
+    field_messages: conversations
+    message_property_mappings:
+      role: from
+      content: value
+
+  - path: youjunhyeok/smoltalk-ko-translate
+    type: chat_template
+    split: train
+    name: merge_filtered
+    field_messages: conversations
+    message_property_mappings:
+      role: role
+      content: content
+
 dataset_prepared_path: last_run_prepared
 val_set_size: 0.001
 save_safetensors: true
@@ -94,9 +168,9 @@ fsdp_config:
 
 # tiny-ko-124m-sft
 
-This model is a fine-tuned version of [minpeter/tiny-ko-124m-base](https://huggingface.co/minpeter/tiny-ko-124m-base) on the lemon-mint/smol-koreantalk dataset.
+This model is a fine-tuned version of [minpeter/tiny-ko-124m-base](https://huggingface.co/minpeter/tiny-ko-124m-base) on the lemon-mint/Korean-FineTome-100k, the lemon-mint/smol-koreantalk, the heegyu/open-korean-instructions-v20231020, the trillionlabs/multisystem-curated, the allenai/tulu-3-sft-personas-instruction-following, the coastral/korean-writing-style-instruct, the devngho/korean-instruction-mix, the youjunhyeok/Magpie-Pro-300K-Filtered-ko and the youjunhyeok/smoltalk-ko-translate datasets.
 It achieves the following results on the evaluation set:
-- Loss: 1.8151
+- Loss: 1.7098
 
 ## Model description
 
@@ -127,17 +201,38 @@ The following hyperparameters were used during training:
 - optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 20
-- training_steps: 887
+- training_steps: 5042
 
 ### Training results
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| No log        | 0      | 0    | 2.8035          |
-| 2.0195        | 0.2256 | 200  | 1.9871          |
-| 1.8857        | 0.4513 | 400  | 1.8815          |
-| 1.8013        | 0.6769 | 600  | 1.8270          |
-| 1.8489        | 0.9026 | 800  | 1.8151          |
+| No log        | 0      | 0    | 2.7016          |
+| 2.1419        | 0.0397 | 200  | 2.1320          |
+| 2.0675        | 0.0793 | 400  | 2.0446          |
+| 2.0252        | 0.1190 | 600  | 1.9864          |
+| 1.9304        | 0.1587 | 800  | 1.9468          |
+| 1.9536        | 0.1983 | 1000 | 1.9145          |
+| 1.8692        | 0.2380 | 1200 | 1.8879          |
+| 1.8556        | 0.2777 | 1400 | 1.8645          |
+| 1.8421        | 0.3174 | 1600 | 1.8433          |
+| 1.9118        | 0.3570 | 1800 | 1.8256          |
+| 1.7791        | 0.3967 | 2000 | 1.8090          |
+| 1.8162        | 0.4364 | 2200 | 1.7934          |
+| 1.796         | 0.4760 | 2400 | 1.7795          |
+| 1.749         | 0.5157 | 2600 | 1.7661          |
+| 1.7536        | 0.5554 | 2800 | 1.7540          |
+| 1.7672        | 0.5950 | 3000 | 1.7432          |
+| 1.7523        | 0.6347 | 3200 | 1.7336          |
+| 1.7074        | 0.6744 | 3400 | 1.7259          |
+| 1.7218        | 0.7141 | 3600 | 1.7202          |
+| 1.6928        | 0.7537 | 3800 | 1.7158          |
+| 1.7184        | 0.7934 | 4000 | 1.7127          |
+| 1.761         | 0.8331 | 4200 | 1.7109          |
+| 1.7481        | 0.8727 | 4400 | 1.7101          |
+| 1.7245        | 0.9124 | 4600 | 1.7098          |
+| 1.7076        | 0.9521 | 4800 | 1.7097          |
+| 1.7403        | 0.9917 | 5000 | 1.7098          |
 
 
 ### Framework versions