diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c215c811652bb814bf6edd1c085421981e282e0b --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: transformers +model_name: Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1 +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1 + +This model is a fine-tuned version of [Qwen/Qwen3-14B-Base](https://huggingface.co/Qwen/Qwen3-14B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/tjg90wvc) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0348981a57e12ef7a9848b8e844e6e0fe0055fc6 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: transformers +model_name: Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2 +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2 + +This model is a fine-tuned version of [Qwen/Qwen3-14B-Base](https://huggingface.co/Qwen/Qwen3-14B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/3ulga1iu) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cd0d97a8c611bcad7d4624b6232e1767e61fe6d3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.00237968804112545, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..01574a85885a2d7e99a61ed87b73b3e6d8294e3c --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/trainer_state.json @@ -0,0 +1,297 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1167, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6110110306739807, + "epoch": 0.1287001287001287, + "grad_norm": 0.8611119389533997, + "learning_rate": 3.4130257962133866e-05, + "loss": 1.53956298828125, + "mean_token_accuracy": 0.6652342769503593, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.8662820833921433, + "epoch": 0.2574002574002574, + "grad_norm": 0.7405035495758057, + "learning_rate": 6.895705180104598e-05, + "loss": 0.7929539489746094, + "mean_token_accuracy": 0.7786347842216492, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7912819278240204, + "epoch": 0.3861003861003861, + "grad_norm": 0.6105485558509827, + "learning_rate": 0.00010378384563995809, + "loss": 0.7236511993408203, + "mean_token_accuracy": 0.7925467795133591, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7777709531784057, + "epoch": 0.5148005148005148, + "grad_norm": 0.5781793594360352, + "learning_rate": 0.0001386106394788702, + "loss": 0.7024919891357422, + "mean_token_accuracy": 0.7969876372814179, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7632609683275223, + "epoch": 0.6435006435006435, + "grad_norm": 0.6553444862365723, + "learning_rate": 0.00017343743331778232, + "loss": 0.7000718688964844, + "mean_token_accuracy": 0.7994742071628571, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.754165632724762, + "epoch": 0.7722007722007722, + "grad_norm": 0.634222149848938, + "learning_rate": 0.00020826422715669444, + "loss": 0.6909049987792969, + "mean_token_accuracy": 0.8016809666156769, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.7530711203813553, + "epoch": 0.9009009009009009, + "grad_norm": 0.7020156383514404, + "learning_rate": 0.00024309102099560653, + "loss": 0.69554931640625, + "mean_token_accuracy": 0.8002807641029358, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.5728044164242204, + "eval_loss": 0.6785851716995239, + "eval_mean_token_accuracy": 0.798433246993527, + "eval_num_tokens": 555493.0, + "eval_runtime": 162.4129, + "eval_samples_per_second": 9.513, + "eval_steps_per_second": 1.194, + "step": 389 + }, + { + "entropy": 0.7489313946829902, + "epoch": 1.0283140283140284, + "grad_norm": 0.7532815933227539, + "learning_rate": 0.0002709470016827303, + "loss": 0.6856581878662109, + "mean_token_accuracy": 0.8023027079273956, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7238409864902496, + "epoch": 1.157014157014157, + "grad_norm": 1.2016215324401855, + "learning_rate": 0.0002707561443541359, + "loss": 0.6699818420410156, + "mean_token_accuracy": 0.8070302194356919, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7388938587903976, + "epoch": 1.2857142857142856, + "grad_norm": 0.7279272079467773, + "learning_rate": 0.0002702930068622498, + "loss": 0.6728517150878907, + "mean_token_accuracy": 0.8049580943584442, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7284485149383545, + "epoch": 1.4144144144144144, + "grad_norm": 0.8563987016677856, + "learning_rate": 0.0002695585213716931, + "loss": 0.6657986450195312, + "mean_token_accuracy": 0.8085588800907135, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6960959500074386, + "epoch": 1.5431145431145432, + "grad_norm": 0.5145474672317505, + "learning_rate": 0.0002685541661937683, + "loss": 0.6358638763427734, + "mean_token_accuracy": 0.8131621342897415, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.7220414417982102, + "epoch": 1.6718146718146718, + "grad_norm": 0.9381059408187866, + "learning_rate": 0.00026728196281103746, + "loss": 0.6531407928466797, + "mean_token_accuracy": 0.811619822382927, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6955459499359131, + "epoch": 1.8005148005148004, + "grad_norm": 0.6115108728408813, + "learning_rate": 0.0002657444718086503, + "loss": 0.6269588088989257, + "mean_token_accuracy": 0.8155373805761337, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6929812705516816, + "epoch": 1.9292149292149292, + "grad_norm": 0.749489426612854, + "learning_rate": 0.0002639447877206115, + "loss": 0.629054069519043, + "mean_token_accuracy": 0.8183021235466004, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5833694102223387, + "eval_loss": 0.6328718662261963, + "eval_mean_token_accuracy": 0.8159044071571114, + "eval_num_tokens": 1110986.0, + "eval_runtime": 161.885, + "eval_samples_per_second": 9.544, + "eval_steps_per_second": 1.198, + "step": 778 + }, + { + "entropy": 0.647241060480927, + "epoch": 2.056628056628057, + "grad_norm": 0.7070767879486084, + "learning_rate": 0.00026188653280135975, + "loss": 0.5823922348022461, + "mean_token_accuracy": 0.8265365301960647, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5995378407835961, + "epoch": 2.1853281853281854, + "grad_norm": 0.8090486526489258, + "learning_rate": 0.0002595738497351955, + "loss": 0.5325597763061524, + "mean_token_accuracy": 0.8369336777925491, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.6043145382404327, + "epoch": 2.314028314028314, + "grad_norm": 0.8279913067817688, + "learning_rate": 0.00025701139329823054, + "loss": 0.5414446258544922, + "mean_token_accuracy": 0.8361396533250809, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5953224584460258, + "epoch": 2.4427284427284426, + "grad_norm": 0.6075023412704468, + "learning_rate": 0.00025420432098964183, + "loss": 0.536654167175293, + "mean_token_accuracy": 0.8340093129873276, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5998479858040809, + "epoch": 2.571428571428571, + "grad_norm": 1.0311471223831177, + "learning_rate": 0.0002511582826510862, + "loss": 0.5372924423217773, + "mean_token_accuracy": 0.8366045409440994, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5980158120393753, + "epoch": 2.7001287001287, + "grad_norm": 0.5971426367759705, + "learning_rate": 0.0002478794090951689, + "loss": 0.5392885208129883, + "mean_token_accuracy": 0.8347727072238922, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.5990680930018425, + "epoch": 2.828828828828829, + "grad_norm": 0.5662627220153809, + "learning_rate": 0.0002443742997658538, + "loss": 0.5360498428344727, + "mean_token_accuracy": 0.8371847170591354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5957039377093315, + "epoch": 2.9575289575289574, + "grad_norm": 0.5043798685073853, + "learning_rate": 0.00024065000945565205, + "loss": 0.5342231369018555, + "mean_token_accuracy": 0.8380735236406326, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5346978819861854, + "eval_loss": 0.6255015134811401, + "eval_mean_token_accuracy": 0.8202014476368108, + "eval_num_tokens": 1666479.0, + "eval_runtime": 161.6098, + "eval_samples_per_second": 9.56, + "eval_steps_per_second": 1.2, + "step": 1167 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.7917443648582656e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cd0d97a8c611bcad7d4624b6232e1767e61fe6d3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.00237968804112545, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2dd54b2318f585587d8ac563bbb57cc8270733f3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/trainer_state.json @@ -0,0 +1,388 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1556, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6110110306739807, + "epoch": 0.1287001287001287, + "grad_norm": 0.8611119389533997, + "learning_rate": 3.4130257962133866e-05, + "loss": 1.53956298828125, + "mean_token_accuracy": 0.6652342769503593, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.8662820833921433, + "epoch": 0.2574002574002574, + "grad_norm": 0.7405035495758057, + "learning_rate": 6.895705180104598e-05, + "loss": 0.7929539489746094, + "mean_token_accuracy": 0.7786347842216492, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7912819278240204, + "epoch": 0.3861003861003861, + "grad_norm": 0.6105485558509827, + "learning_rate": 0.00010378384563995809, + "loss": 0.7236511993408203, + "mean_token_accuracy": 0.7925467795133591, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7777709531784057, + "epoch": 0.5148005148005148, + "grad_norm": 0.5781793594360352, + "learning_rate": 0.0001386106394788702, + "loss": 0.7024919891357422, + "mean_token_accuracy": 0.7969876372814179, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7632609683275223, + "epoch": 0.6435006435006435, + "grad_norm": 0.6553444862365723, + "learning_rate": 0.00017343743331778232, + "loss": 0.7000718688964844, + "mean_token_accuracy": 0.7994742071628571, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.754165632724762, + "epoch": 0.7722007722007722, + "grad_norm": 0.634222149848938, + "learning_rate": 0.00020826422715669444, + "loss": 0.6909049987792969, + "mean_token_accuracy": 0.8016809666156769, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.7530711203813553, + "epoch": 0.9009009009009009, + "grad_norm": 0.7020156383514404, + "learning_rate": 0.00024309102099560653, + "loss": 0.69554931640625, + "mean_token_accuracy": 0.8002807641029358, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.5728044164242204, + "eval_loss": 0.6785851716995239, + "eval_mean_token_accuracy": 0.798433246993527, + "eval_num_tokens": 555493.0, + "eval_runtime": 162.4129, + "eval_samples_per_second": 9.513, + "eval_steps_per_second": 1.194, + "step": 389 + }, + { + "entropy": 0.7489313946829902, + "epoch": 1.0283140283140284, + "grad_norm": 0.7532815933227539, + "learning_rate": 0.0002709470016827303, + "loss": 0.6856581878662109, + "mean_token_accuracy": 0.8023027079273956, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7238409864902496, + "epoch": 1.157014157014157, + "grad_norm": 1.2016215324401855, + "learning_rate": 0.0002707561443541359, + "loss": 0.6699818420410156, + "mean_token_accuracy": 0.8070302194356919, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7388938587903976, + "epoch": 1.2857142857142856, + "grad_norm": 0.7279272079467773, + "learning_rate": 0.0002702930068622498, + "loss": 0.6728517150878907, + "mean_token_accuracy": 0.8049580943584442, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7284485149383545, + "epoch": 1.4144144144144144, + "grad_norm": 0.8563987016677856, + "learning_rate": 0.0002695585213716931, + "loss": 0.6657986450195312, + "mean_token_accuracy": 0.8085588800907135, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6960959500074386, + "epoch": 1.5431145431145432, + "grad_norm": 0.5145474672317505, + "learning_rate": 0.0002685541661937683, + "loss": 0.6358638763427734, + "mean_token_accuracy": 0.8131621342897415, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.7220414417982102, + "epoch": 1.6718146718146718, + "grad_norm": 0.9381059408187866, + "learning_rate": 0.00026728196281103746, + "loss": 0.6531407928466797, + "mean_token_accuracy": 0.811619822382927, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6955459499359131, + "epoch": 1.8005148005148004, + "grad_norm": 0.6115108728408813, + "learning_rate": 0.0002657444718086503, + "loss": 0.6269588088989257, + "mean_token_accuracy": 0.8155373805761337, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6929812705516816, + "epoch": 1.9292149292149292, + "grad_norm": 0.749489426612854, + "learning_rate": 0.0002639447877206115, + "loss": 0.629054069519043, + "mean_token_accuracy": 0.8183021235466004, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5833694102223387, + "eval_loss": 0.6328718662261963, + "eval_mean_token_accuracy": 0.8159044071571114, + "eval_num_tokens": 1110986.0, + "eval_runtime": 161.885, + "eval_samples_per_second": 9.544, + "eval_steps_per_second": 1.198, + "step": 778 + }, + { + "entropy": 0.647241060480927, + "epoch": 2.056628056628057, + "grad_norm": 0.7070767879486084, + "learning_rate": 0.00026188653280135975, + "loss": 0.5823922348022461, + "mean_token_accuracy": 0.8265365301960647, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5995378407835961, + "epoch": 2.1853281853281854, + "grad_norm": 0.8090486526489258, + "learning_rate": 0.0002595738497351955, + "loss": 0.5325597763061524, + "mean_token_accuracy": 0.8369336777925491, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.6043145382404327, + "epoch": 2.314028314028314, + "grad_norm": 0.8279913067817688, + "learning_rate": 0.00025701139329823054, + "loss": 0.5414446258544922, + "mean_token_accuracy": 0.8361396533250809, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5953224584460258, + "epoch": 2.4427284427284426, + "grad_norm": 0.6075023412704468, + "learning_rate": 0.00025420432098964183, + "loss": 0.536654167175293, + "mean_token_accuracy": 0.8340093129873276, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5998479858040809, + "epoch": 2.571428571428571, + "grad_norm": 1.0311471223831177, + "learning_rate": 0.0002511582826510862, + "loss": 0.5372924423217773, + "mean_token_accuracy": 0.8366045409440994, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5980158120393753, + "epoch": 2.7001287001287, + "grad_norm": 0.5971426367759705, + "learning_rate": 0.0002478794090951689, + "loss": 0.5392885208129883, + "mean_token_accuracy": 0.8347727072238922, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.5990680930018425, + "epoch": 2.828828828828829, + "grad_norm": 0.5662627220153809, + "learning_rate": 0.0002443742997658538, + "loss": 0.5360498428344727, + "mean_token_accuracy": 0.8371847170591354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5957039377093315, + "epoch": 2.9575289575289574, + "grad_norm": 0.5043798685073853, + "learning_rate": 0.00024065000945565205, + "loss": 0.5342231369018555, + "mean_token_accuracy": 0.8380735236406326, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5346978819861854, + "eval_loss": 0.6255015134811401, + "eval_mean_token_accuracy": 0.8202014476368108, + "eval_num_tokens": 1666479.0, + "eval_runtime": 161.6098, + "eval_samples_per_second": 9.56, + "eval_steps_per_second": 1.2, + "step": 1167 + }, + { + "entropy": 0.5212209137401196, + "epoch": 3.0849420849420848, + "grad_norm": 0.7095440626144409, + "learning_rate": 0.00023671403410632178, + "loss": 0.45311901092529294, + "mean_token_accuracy": 0.856362871449403, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.4701593083143234, + "epoch": 3.213642213642214, + "grad_norm": 0.6408083438873291, + "learning_rate": 0.0002325742957216607, + "loss": 0.39916397094726563, + "mean_token_accuracy": 0.8698061722517013, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4766591975092888, + "epoch": 3.3423423423423424, + "grad_norm": 0.6415093541145325, + "learning_rate": 0.0002282391264227552, + "loss": 0.4116698455810547, + "mean_token_accuracy": 0.8679435575008392, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.4937947469949722, + "epoch": 3.471042471042471, + "grad_norm": 0.6549825072288513, + "learning_rate": 0.00022371725167778054, + "loss": 0.4296376037597656, + "mean_token_accuracy": 0.8609357953071595, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.4920153194665909, + "epoch": 3.5997425997425996, + "grad_norm": 0.6452126502990723, + "learning_rate": 0.00021901777274010406, + "loss": 0.4307489013671875, + "mean_token_accuracy": 0.8606827831268311, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.490042342543602, + "epoch": 3.7284427284427286, + "grad_norm": 0.5727734565734863, + "learning_rate": 0.0002141501483300395, + "loss": 0.4295254135131836, + "mean_token_accuracy": 0.8616545403003693, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.49857193052768706, + "epoch": 3.857142857142857, + "grad_norm": 0.7732954025268555, + "learning_rate": 0.00020912417559712133, + "loss": 0.4289303207397461, + "mean_token_accuracy": 0.8616443765163422, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.4746784272789955, + "epoch": 3.985842985842986, + "grad_norm": 0.5791187882423401, + "learning_rate": 0.00020394997040121726, + "loss": 0.4180263900756836, + "mean_token_accuracy": 0.866080379486084, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4793926059585257, + "eval_loss": 0.627627968788147, + "eval_mean_token_accuracy": 0.8233870095813397, + "eval_num_tokens": 2221972.0, + "eval_runtime": 162.0225, + "eval_samples_per_second": 9.536, + "eval_steps_per_second": 1.197, + "step": 1556 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.723284547240653e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cd0d97a8c611bcad7d4624b6232e1767e61fe6d3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.00237968804112545, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..cb9ecca8fcbd02a19096687fb29b85701575873a --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/trainer_state.json @@ -0,0 +1,469 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 1945, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6110110306739807, + "epoch": 0.1287001287001287, + "grad_norm": 0.8611119389533997, + "learning_rate": 3.4130257962133866e-05, + "loss": 1.53956298828125, + "mean_token_accuracy": 0.6652342769503593, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.8662820833921433, + "epoch": 0.2574002574002574, + "grad_norm": 0.7405035495758057, + "learning_rate": 6.895705180104598e-05, + "loss": 0.7929539489746094, + "mean_token_accuracy": 0.7786347842216492, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7912819278240204, + "epoch": 0.3861003861003861, + "grad_norm": 0.6105485558509827, + "learning_rate": 0.00010378384563995809, + "loss": 0.7236511993408203, + "mean_token_accuracy": 0.7925467795133591, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7777709531784057, + "epoch": 0.5148005148005148, + "grad_norm": 0.5781793594360352, + "learning_rate": 0.0001386106394788702, + "loss": 0.7024919891357422, + "mean_token_accuracy": 0.7969876372814179, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7632609683275223, + "epoch": 0.6435006435006435, + "grad_norm": 0.6553444862365723, + "learning_rate": 0.00017343743331778232, + "loss": 0.7000718688964844, + "mean_token_accuracy": 0.7994742071628571, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.754165632724762, + "epoch": 0.7722007722007722, + "grad_norm": 0.634222149848938, + "learning_rate": 0.00020826422715669444, + "loss": 0.6909049987792969, + "mean_token_accuracy": 0.8016809666156769, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.7530711203813553, + "epoch": 0.9009009009009009, + "grad_norm": 0.7020156383514404, + "learning_rate": 0.00024309102099560653, + "loss": 0.69554931640625, + "mean_token_accuracy": 0.8002807641029358, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.5728044164242204, + "eval_loss": 0.6785851716995239, + "eval_mean_token_accuracy": 0.798433246993527, + "eval_num_tokens": 555493.0, + "eval_runtime": 162.4129, + "eval_samples_per_second": 9.513, + "eval_steps_per_second": 1.194, + "step": 389 + }, + { + "entropy": 0.7489313946829902, + "epoch": 1.0283140283140284, + "grad_norm": 0.7532815933227539, + "learning_rate": 0.0002709470016827303, + "loss": 0.6856581878662109, + "mean_token_accuracy": 0.8023027079273956, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7238409864902496, + "epoch": 1.157014157014157, + "grad_norm": 1.2016215324401855, + "learning_rate": 0.0002707561443541359, + "loss": 0.6699818420410156, + "mean_token_accuracy": 0.8070302194356919, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7388938587903976, + "epoch": 1.2857142857142856, + "grad_norm": 0.7279272079467773, + "learning_rate": 0.0002702930068622498, + "loss": 0.6728517150878907, + "mean_token_accuracy": 0.8049580943584442, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7284485149383545, + "epoch": 1.4144144144144144, + "grad_norm": 0.8563987016677856, + "learning_rate": 0.0002695585213716931, + "loss": 0.6657986450195312, + "mean_token_accuracy": 0.8085588800907135, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6960959500074386, + "epoch": 1.5431145431145432, + "grad_norm": 0.5145474672317505, + "learning_rate": 0.0002685541661937683, + "loss": 0.6358638763427734, + "mean_token_accuracy": 0.8131621342897415, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.7220414417982102, + "epoch": 1.6718146718146718, + "grad_norm": 0.9381059408187866, + "learning_rate": 0.00026728196281103746, + "loss": 0.6531407928466797, + "mean_token_accuracy": 0.811619822382927, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6955459499359131, + "epoch": 1.8005148005148004, + "grad_norm": 0.6115108728408813, + "learning_rate": 0.0002657444718086503, + "loss": 0.6269588088989257, + "mean_token_accuracy": 0.8155373805761337, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6929812705516816, + "epoch": 1.9292149292149292, + "grad_norm": 0.749489426612854, + "learning_rate": 0.0002639447877206115, + "loss": 0.629054069519043, + "mean_token_accuracy": 0.8183021235466004, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5833694102223387, + "eval_loss": 0.6328718662261963, + "eval_mean_token_accuracy": 0.8159044071571114, + "eval_num_tokens": 1110986.0, + "eval_runtime": 161.885, + "eval_samples_per_second": 9.544, + "eval_steps_per_second": 1.198, + "step": 778 + }, + { + "entropy": 0.647241060480927, + "epoch": 2.056628056628057, + "grad_norm": 0.7070767879486084, + "learning_rate": 0.00026188653280135975, + "loss": 0.5823922348022461, + "mean_token_accuracy": 0.8265365301960647, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5995378407835961, + "epoch": 2.1853281853281854, + "grad_norm": 0.8090486526489258, + "learning_rate": 0.0002595738497351955, + "loss": 0.5325597763061524, + "mean_token_accuracy": 0.8369336777925491, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.6043145382404327, + "epoch": 2.314028314028314, + "grad_norm": 0.8279913067817688, + "learning_rate": 0.00025701139329823054, + "loss": 0.5414446258544922, + "mean_token_accuracy": 0.8361396533250809, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5953224584460258, + "epoch": 2.4427284427284426, + "grad_norm": 0.6075023412704468, + "learning_rate": 0.00025420432098964183, + "loss": 0.536654167175293, + "mean_token_accuracy": 0.8340093129873276, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5998479858040809, + "epoch": 2.571428571428571, + "grad_norm": 1.0311471223831177, + "learning_rate": 0.0002511582826510862, + "loss": 0.5372924423217773, + "mean_token_accuracy": 0.8366045409440994, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5980158120393753, + "epoch": 2.7001287001287, + "grad_norm": 0.5971426367759705, + "learning_rate": 0.0002478794090951689, + "loss": 0.5392885208129883, + "mean_token_accuracy": 0.8347727072238922, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.5990680930018425, + "epoch": 2.828828828828829, + "grad_norm": 0.5662627220153809, + "learning_rate": 0.0002443742997658538, + "loss": 0.5360498428344727, + "mean_token_accuracy": 0.8371847170591354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5957039377093315, + "epoch": 2.9575289575289574, + "grad_norm": 0.5043798685073853, + "learning_rate": 0.00024065000945565205, + "loss": 0.5342231369018555, + "mean_token_accuracy": 0.8380735236406326, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5346978819861854, + "eval_loss": 0.6255015134811401, + "eval_mean_token_accuracy": 0.8202014476368108, + "eval_num_tokens": 1666479.0, + "eval_runtime": 161.6098, + "eval_samples_per_second": 9.56, + "eval_steps_per_second": 1.2, + "step": 1167 + }, + { + "entropy": 0.5212209137401196, + "epoch": 3.0849420849420848, + "grad_norm": 0.7095440626144409, + "learning_rate": 0.00023671403410632178, + "loss": 0.45311901092529294, + "mean_token_accuracy": 0.856362871449403, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.4701593083143234, + "epoch": 3.213642213642214, + "grad_norm": 0.6408083438873291, + "learning_rate": 0.0002325742957216607, + "loss": 0.39916397094726563, + "mean_token_accuracy": 0.8698061722517013, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4766591975092888, + "epoch": 3.3423423423423424, + "grad_norm": 0.6415093541145325, + "learning_rate": 0.0002282391264227552, + "loss": 0.4116698455810547, + "mean_token_accuracy": 0.8679435575008392, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.4937947469949722, + "epoch": 3.471042471042471, + "grad_norm": 0.6549825072288513, + "learning_rate": 0.00022371725167778054, + "loss": 0.4296376037597656, + "mean_token_accuracy": 0.8609357953071595, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.4920153194665909, + "epoch": 3.5997425997425996, + "grad_norm": 0.6452126502990723, + "learning_rate": 0.00021901777274010406, + "loss": 0.4307489013671875, + "mean_token_accuracy": 0.8606827831268311, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.490042342543602, + "epoch": 3.7284427284427286, + "grad_norm": 0.5727734565734863, + "learning_rate": 0.0002141501483300395, + "loss": 0.4295254135131836, + "mean_token_accuracy": 0.8616545403003693, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.49857193052768706, + "epoch": 3.857142857142857, + "grad_norm": 0.7732954025268555, + "learning_rate": 0.00020912417559712133, + "loss": 0.4289303207397461, + "mean_token_accuracy": 0.8616443765163422, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.4746784272789955, + "epoch": 3.985842985842986, + "grad_norm": 0.5791187882423401, + "learning_rate": 0.00020394997040121726, + "loss": 0.4180263900756836, + "mean_token_accuracy": 0.866080379486084, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4793926059585257, + "eval_loss": 0.627627968788147, + "eval_mean_token_accuracy": 0.8233870095813397, + "eval_num_tokens": 2221972.0, + "eval_runtime": 162.0225, + "eval_samples_per_second": 9.536, + "eval_steps_per_second": 1.197, + "step": 1556 + }, + { + "entropy": 0.38195489000792454, + "epoch": 4.113256113256114, + "grad_norm": 0.6002617478370667, + "learning_rate": 0.0001986379469521669, + "loss": 0.30819049835205076, + "mean_token_accuracy": 0.8977848634575353, + "num_tokens": 2282164.0, + "step": 1600 + }, + { + "entropy": 0.3655787402391434, + "epoch": 4.241956241956242, + "grad_norm": 0.7100041508674622, + "learning_rate": 0.00019319879684892634, + "loss": 0.29959835052490236, + "mean_token_accuracy": 0.8991208010911942, + "num_tokens": 2353213.0, + "step": 1650 + }, + { + "entropy": 0.3821141055226326, + "epoch": 4.370656370656371, + "grad_norm": 0.5848307013511658, + "learning_rate": 0.00018764346756040715, + "loss": 0.313802490234375, + "mean_token_accuracy": 0.895167955160141, + "num_tokens": 2425068.0, + "step": 1700 + }, + { + "entropy": 0.37083797007799146, + "epoch": 4.499356499356499, + "grad_norm": 0.6447024941444397, + "learning_rate": 0.00018198314039132143, + "loss": 0.30583988189697264, + "mean_token_accuracy": 0.8961733293533325, + "num_tokens": 2498321.0, + "step": 1750 + }, + { + "entropy": 0.3791545969247818, + "epoch": 4.628056628056628, + "grad_norm": 0.6575382351875305, + "learning_rate": 0.00017622920797738184, + "loss": 0.3088031005859375, + "mean_token_accuracy": 0.8960050916671753, + "num_tokens": 2570321.0, + "step": 1800 + }, + { + "entropy": 0.3946831756830215, + "epoch": 4.756756756756757, + "grad_norm": 0.5351552963256836, + "learning_rate": 0.00017039325135515207, + "loss": 0.3229162979125977, + "mean_token_accuracy": 0.8920552498102188, + "num_tokens": 2642851.0, + "step": 1850 + }, + { + "entropy": 0.37298239797353744, + "epoch": 4.885456885456885, + "grad_norm": 0.7624587416648865, + "learning_rate": 0.00016448701665269964, + "loss": 0.3067934799194336, + "mean_token_accuracy": 0.8951873427629471, + "num_tokens": 2715629.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3738150204887095, + "eval_loss": 0.7131896615028381, + "eval_mean_token_accuracy": 0.8207752468045225, + "eval_num_tokens": 2777465.0, + "eval_runtime": 162.1244, + "eval_samples_per_second": 9.53, + "eval_steps_per_second": 1.197, + "step": 1945 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.653233039031091e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cd0d97a8c611bcad7d4624b6232e1767e61fe6d3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.00237968804112545, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b513891c43cf49de1d61c1add60b3166eea1c0b3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/trainer_state.json @@ -0,0 +1,560 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2334, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6110110306739807, + "epoch": 0.1287001287001287, + "grad_norm": 0.8611119389533997, + "learning_rate": 3.4130257962133866e-05, + "loss": 1.53956298828125, + "mean_token_accuracy": 0.6652342769503593, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.8662820833921433, + "epoch": 0.2574002574002574, + "grad_norm": 0.7405035495758057, + "learning_rate": 6.895705180104598e-05, + "loss": 0.7929539489746094, + "mean_token_accuracy": 0.7786347842216492, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7912819278240204, + "epoch": 0.3861003861003861, + "grad_norm": 0.6105485558509827, + "learning_rate": 0.00010378384563995809, + "loss": 0.7236511993408203, + "mean_token_accuracy": 0.7925467795133591, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7777709531784057, + "epoch": 0.5148005148005148, + "grad_norm": 0.5781793594360352, + "learning_rate": 0.0001386106394788702, + "loss": 0.7024919891357422, + "mean_token_accuracy": 0.7969876372814179, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7632609683275223, + "epoch": 0.6435006435006435, + "grad_norm": 0.6553444862365723, + "learning_rate": 0.00017343743331778232, + "loss": 0.7000718688964844, + "mean_token_accuracy": 0.7994742071628571, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.754165632724762, + "epoch": 0.7722007722007722, + "grad_norm": 0.634222149848938, + "learning_rate": 0.00020826422715669444, + "loss": 0.6909049987792969, + "mean_token_accuracy": 0.8016809666156769, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.7530711203813553, + "epoch": 0.9009009009009009, + "grad_norm": 0.7020156383514404, + "learning_rate": 0.00024309102099560653, + "loss": 0.69554931640625, + "mean_token_accuracy": 0.8002807641029358, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.5728044164242204, + "eval_loss": 0.6785851716995239, + "eval_mean_token_accuracy": 0.798433246993527, + "eval_num_tokens": 555493.0, + "eval_runtime": 162.4129, + "eval_samples_per_second": 9.513, + "eval_steps_per_second": 1.194, + "step": 389 + }, + { + "entropy": 0.7489313946829902, + "epoch": 1.0283140283140284, + "grad_norm": 0.7532815933227539, + "learning_rate": 0.0002709470016827303, + "loss": 0.6856581878662109, + "mean_token_accuracy": 0.8023027079273956, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7238409864902496, + "epoch": 1.157014157014157, + "grad_norm": 1.2016215324401855, + "learning_rate": 0.0002707561443541359, + "loss": 0.6699818420410156, + "mean_token_accuracy": 0.8070302194356919, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7388938587903976, + "epoch": 1.2857142857142856, + "grad_norm": 0.7279272079467773, + "learning_rate": 0.0002702930068622498, + "loss": 0.6728517150878907, + "mean_token_accuracy": 0.8049580943584442, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7284485149383545, + "epoch": 1.4144144144144144, + "grad_norm": 0.8563987016677856, + "learning_rate": 0.0002695585213716931, + "loss": 0.6657986450195312, + "mean_token_accuracy": 0.8085588800907135, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6960959500074386, + "epoch": 1.5431145431145432, + "grad_norm": 0.5145474672317505, + "learning_rate": 0.0002685541661937683, + "loss": 0.6358638763427734, + "mean_token_accuracy": 0.8131621342897415, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.7220414417982102, + "epoch": 1.6718146718146718, + "grad_norm": 0.9381059408187866, + "learning_rate": 0.00026728196281103746, + "loss": 0.6531407928466797, + "mean_token_accuracy": 0.811619822382927, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6955459499359131, + "epoch": 1.8005148005148004, + "grad_norm": 0.6115108728408813, + "learning_rate": 0.0002657444718086503, + "loss": 0.6269588088989257, + "mean_token_accuracy": 0.8155373805761337, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6929812705516816, + "epoch": 1.9292149292149292, + "grad_norm": 0.749489426612854, + "learning_rate": 0.0002639447877206115, + "loss": 0.629054069519043, + "mean_token_accuracy": 0.8183021235466004, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5833694102223387, + "eval_loss": 0.6328718662261963, + "eval_mean_token_accuracy": 0.8159044071571114, + "eval_num_tokens": 1110986.0, + "eval_runtime": 161.885, + "eval_samples_per_second": 9.544, + "eval_steps_per_second": 1.198, + "step": 778 + }, + { + "entropy": 0.647241060480927, + "epoch": 2.056628056628057, + "grad_norm": 0.7070767879486084, + "learning_rate": 0.00026188653280135975, + "loss": 0.5823922348022461, + "mean_token_accuracy": 0.8265365301960647, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5995378407835961, + "epoch": 2.1853281853281854, + "grad_norm": 0.8090486526489258, + "learning_rate": 0.0002595738497351955, + "loss": 0.5325597763061524, + "mean_token_accuracy": 0.8369336777925491, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.6043145382404327, + "epoch": 2.314028314028314, + "grad_norm": 0.8279913067817688, + "learning_rate": 0.00025701139329823054, + "loss": 0.5414446258544922, + "mean_token_accuracy": 0.8361396533250809, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5953224584460258, + "epoch": 2.4427284427284426, + "grad_norm": 0.6075023412704468, + "learning_rate": 0.00025420432098964183, + "loss": 0.536654167175293, + "mean_token_accuracy": 0.8340093129873276, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5998479858040809, + "epoch": 2.571428571428571, + "grad_norm": 1.0311471223831177, + "learning_rate": 0.0002511582826510862, + "loss": 0.5372924423217773, + "mean_token_accuracy": 0.8366045409440994, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5980158120393753, + "epoch": 2.7001287001287, + "grad_norm": 0.5971426367759705, + "learning_rate": 0.0002478794090951689, + "loss": 0.5392885208129883, + "mean_token_accuracy": 0.8347727072238922, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.5990680930018425, + "epoch": 2.828828828828829, + "grad_norm": 0.5662627220153809, + "learning_rate": 0.0002443742997658538, + "loss": 0.5360498428344727, + "mean_token_accuracy": 0.8371847170591354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5957039377093315, + "epoch": 2.9575289575289574, + "grad_norm": 0.5043798685073853, + "learning_rate": 0.00024065000945565205, + "loss": 0.5342231369018555, + "mean_token_accuracy": 0.8380735236406326, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5346978819861854, + "eval_loss": 0.6255015134811401, + "eval_mean_token_accuracy": 0.8202014476368108, + "eval_num_tokens": 1666479.0, + "eval_runtime": 161.6098, + "eval_samples_per_second": 9.56, + "eval_steps_per_second": 1.2, + "step": 1167 + }, + { + "entropy": 0.5212209137401196, + "epoch": 3.0849420849420848, + "grad_norm": 0.7095440626144409, + "learning_rate": 0.00023671403410632178, + "loss": 0.45311901092529294, + "mean_token_accuracy": 0.856362871449403, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.4701593083143234, + "epoch": 3.213642213642214, + "grad_norm": 0.6408083438873291, + "learning_rate": 0.0002325742957216607, + "loss": 0.39916397094726563, + "mean_token_accuracy": 0.8698061722517013, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4766591975092888, + "epoch": 3.3423423423423424, + "grad_norm": 0.6415093541145325, + "learning_rate": 0.0002282391264227552, + "loss": 0.4116698455810547, + "mean_token_accuracy": 0.8679435575008392, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.4937947469949722, + "epoch": 3.471042471042471, + "grad_norm": 0.6549825072288513, + "learning_rate": 0.00022371725167778054, + "loss": 0.4296376037597656, + "mean_token_accuracy": 0.8609357953071595, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.4920153194665909, + "epoch": 3.5997425997425996, + "grad_norm": 0.6452126502990723, + "learning_rate": 0.00021901777274010406, + "loss": 0.4307489013671875, + "mean_token_accuracy": 0.8606827831268311, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.490042342543602, + "epoch": 3.7284427284427286, + "grad_norm": 0.5727734565734863, + "learning_rate": 0.0002141501483300395, + "loss": 0.4295254135131836, + "mean_token_accuracy": 0.8616545403003693, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.49857193052768706, + "epoch": 3.857142857142857, + "grad_norm": 0.7732954025268555, + "learning_rate": 0.00020912417559712133, + "loss": 0.4289303207397461, + "mean_token_accuracy": 0.8616443765163422, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.4746784272789955, + "epoch": 3.985842985842986, + "grad_norm": 0.5791187882423401, + "learning_rate": 0.00020394997040121726, + "loss": 0.4180263900756836, + "mean_token_accuracy": 0.866080379486084, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4793926059585257, + "eval_loss": 0.627627968788147, + "eval_mean_token_accuracy": 0.8233870095813397, + "eval_num_tokens": 2221972.0, + "eval_runtime": 162.0225, + "eval_samples_per_second": 9.536, + "eval_steps_per_second": 1.197, + "step": 1556 + }, + { + "entropy": 0.38195489000792454, + "epoch": 4.113256113256114, + "grad_norm": 0.6002617478370667, + "learning_rate": 0.0001986379469521669, + "loss": 0.30819049835205076, + "mean_token_accuracy": 0.8977848634575353, + "num_tokens": 2282164.0, + "step": 1600 + }, + { + "entropy": 0.3655787402391434, + "epoch": 4.241956241956242, + "grad_norm": 0.7100041508674622, + "learning_rate": 0.00019319879684892634, + "loss": 0.29959835052490236, + "mean_token_accuracy": 0.8991208010911942, + "num_tokens": 2353213.0, + "step": 1650 + }, + { + "entropy": 0.3821141055226326, + "epoch": 4.370656370656371, + "grad_norm": 0.5848307013511658, + "learning_rate": 0.00018764346756040715, + "loss": 0.313802490234375, + "mean_token_accuracy": 0.895167955160141, + "num_tokens": 2425068.0, + "step": 1700 + }, + { + "entropy": 0.37083797007799146, + "epoch": 4.499356499356499, + "grad_norm": 0.6447024941444397, + "learning_rate": 0.00018198314039132143, + "loss": 0.30583988189697264, + "mean_token_accuracy": 0.8961733293533325, + "num_tokens": 2498321.0, + "step": 1750 + }, + { + "entropy": 0.3791545969247818, + "epoch": 4.628056628056628, + "grad_norm": 0.6575382351875305, + "learning_rate": 0.00017622920797738184, + "loss": 0.3088031005859375, + "mean_token_accuracy": 0.8960050916671753, + "num_tokens": 2570321.0, + "step": 1800 + }, + { + "entropy": 0.3946831756830215, + "epoch": 4.756756756756757, + "grad_norm": 0.5351552963256836, + "learning_rate": 0.00017039325135515207, + "loss": 0.3229162979125977, + "mean_token_accuracy": 0.8920552498102188, + "num_tokens": 2642851.0, + "step": 1850 + }, + { + "entropy": 0.37298239797353744, + "epoch": 4.885456885456885, + "grad_norm": 0.7624587416648865, + "learning_rate": 0.00016448701665269964, + "loss": 0.3067934799194336, + "mean_token_accuracy": 0.8951873427629471, + "num_tokens": 2715629.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3738150204887095, + "eval_loss": 0.7131896615028381, + "eval_mean_token_accuracy": 0.8207752468045225, + "eval_num_tokens": 2777465.0, + "eval_runtime": 162.1244, + "eval_samples_per_second": 9.53, + "eval_steps_per_second": 1.197, + "step": 1945 + }, + { + "entropy": 0.377820266617669, + "epoch": 5.012870012870013, + "grad_norm": 0.41990190744400024, + "learning_rate": 0.00015852239144796624, + "loss": 0.3058685111999512, + "mean_token_accuracy": 0.8964343480389527, + "num_tokens": 2784896.0, + "step": 1950 + }, + { + "entropy": 0.2714502356946468, + "epoch": 5.141570141570142, + "grad_norm": 0.422568678855896, + "learning_rate": 0.00015251138084243995, + "loss": 0.2093442153930664, + "mean_token_accuracy": 0.9311346983909607, + "num_tokens": 2854374.0, + "step": 2000 + }, + { + "entropy": 0.268475965410471, + "epoch": 5.27027027027027, + "grad_norm": 0.6637414693832397, + "learning_rate": 0.0001464660832982852, + "loss": 0.20736080169677734, + "mean_token_accuracy": 0.9289199805259705, + "num_tokens": 2927362.0, + "step": 2050 + }, + { + "entropy": 0.2644876340031624, + "epoch": 5.398970398970399, + "grad_norm": 0.47317707538604736, + "learning_rate": 0.00014039866628756467, + "loss": 0.20464908599853515, + "mean_token_accuracy": 0.9300856202840805, + "num_tokens": 3000143.0, + "step": 2100 + }, + { + "entropy": 0.2675253136456013, + "epoch": 5.527670527670527, + "grad_norm": 0.5253982543945312, + "learning_rate": 0.00013432134180256338, + "loss": 0.21154335021972656, + "mean_token_accuracy": 0.9283734840154648, + "num_tokens": 3072561.0, + "step": 2150 + }, + { + "entropy": 0.27213907435536383, + "epoch": 5.656370656370656, + "grad_norm": 0.46738553047180176, + "learning_rate": 0.00012824634177650664, + "loss": 0.21339216232299804, + "mean_token_accuracy": 0.9272083270549775, + "num_tokens": 3144831.0, + "step": 2200 + }, + { + "entropy": 0.2785488124191761, + "epoch": 5.785070785070785, + "grad_norm": 0.4469502866268158, + "learning_rate": 0.00012218589346414205, + "loss": 0.21601097106933595, + "mean_token_accuracy": 0.9255663657188415, + "num_tokens": 3215960.0, + "step": 2250 + }, + { + "entropy": 0.2699935150146484, + "epoch": 5.913770913770914, + "grad_norm": 0.7359778881072998, + "learning_rate": 0.00011615219483173828, + "loss": 0.20725584030151367, + "mean_token_accuracy": 0.9286630594730377, + "num_tokens": 3287499.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.26417383682174783, + "eval_loss": 0.8880229592323303, + "eval_mean_token_accuracy": 0.8159987201395723, + "eval_num_tokens": 3332958.0, + "eval_runtime": 162.0991, + "eval_samples_per_second": 9.531, + "eval_steps_per_second": 1.197, + "step": 2334 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.587061113467187e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cd0d97a8c611bcad7d4624b6232e1767e61fe6d3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.00237968804112545, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..771284ca5d55158a2484e04f98269f8c4cc8c463 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/trainer_state.json @@ -0,0 +1,651 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2723, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6110110306739807, + "epoch": 0.1287001287001287, + "grad_norm": 0.8611119389533997, + "learning_rate": 3.4130257962133866e-05, + "loss": 1.53956298828125, + "mean_token_accuracy": 0.6652342769503593, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.8662820833921433, + "epoch": 0.2574002574002574, + "grad_norm": 0.7405035495758057, + "learning_rate": 6.895705180104598e-05, + "loss": 0.7929539489746094, + "mean_token_accuracy": 0.7786347842216492, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7912819278240204, + "epoch": 0.3861003861003861, + "grad_norm": 0.6105485558509827, + "learning_rate": 0.00010378384563995809, + "loss": 0.7236511993408203, + "mean_token_accuracy": 0.7925467795133591, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7777709531784057, + "epoch": 0.5148005148005148, + "grad_norm": 0.5781793594360352, + "learning_rate": 0.0001386106394788702, + "loss": 0.7024919891357422, + "mean_token_accuracy": 0.7969876372814179, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7632609683275223, + "epoch": 0.6435006435006435, + "grad_norm": 0.6553444862365723, + "learning_rate": 0.00017343743331778232, + "loss": 0.7000718688964844, + "mean_token_accuracy": 0.7994742071628571, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.754165632724762, + "epoch": 0.7722007722007722, + "grad_norm": 0.634222149848938, + "learning_rate": 0.00020826422715669444, + "loss": 0.6909049987792969, + "mean_token_accuracy": 0.8016809666156769, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.7530711203813553, + "epoch": 0.9009009009009009, + "grad_norm": 0.7020156383514404, + "learning_rate": 0.00024309102099560653, + "loss": 0.69554931640625, + "mean_token_accuracy": 0.8002807641029358, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.5728044164242204, + "eval_loss": 0.6785851716995239, + "eval_mean_token_accuracy": 0.798433246993527, + "eval_num_tokens": 555493.0, + "eval_runtime": 162.4129, + "eval_samples_per_second": 9.513, + "eval_steps_per_second": 1.194, + "step": 389 + }, + { + "entropy": 0.7489313946829902, + "epoch": 1.0283140283140284, + "grad_norm": 0.7532815933227539, + "learning_rate": 0.0002709470016827303, + "loss": 0.6856581878662109, + "mean_token_accuracy": 0.8023027079273956, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7238409864902496, + "epoch": 1.157014157014157, + "grad_norm": 1.2016215324401855, + "learning_rate": 0.0002707561443541359, + "loss": 0.6699818420410156, + "mean_token_accuracy": 0.8070302194356919, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7388938587903976, + "epoch": 1.2857142857142856, + "grad_norm": 0.7279272079467773, + "learning_rate": 0.0002702930068622498, + "loss": 0.6728517150878907, + "mean_token_accuracy": 0.8049580943584442, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7284485149383545, + "epoch": 1.4144144144144144, + "grad_norm": 0.8563987016677856, + "learning_rate": 0.0002695585213716931, + "loss": 0.6657986450195312, + "mean_token_accuracy": 0.8085588800907135, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6960959500074386, + "epoch": 1.5431145431145432, + "grad_norm": 0.5145474672317505, + "learning_rate": 0.0002685541661937683, + "loss": 0.6358638763427734, + "mean_token_accuracy": 0.8131621342897415, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.7220414417982102, + "epoch": 1.6718146718146718, + "grad_norm": 0.9381059408187866, + "learning_rate": 0.00026728196281103746, + "loss": 0.6531407928466797, + "mean_token_accuracy": 0.811619822382927, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6955459499359131, + "epoch": 1.8005148005148004, + "grad_norm": 0.6115108728408813, + "learning_rate": 0.0002657444718086503, + "loss": 0.6269588088989257, + "mean_token_accuracy": 0.8155373805761337, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6929812705516816, + "epoch": 1.9292149292149292, + "grad_norm": 0.749489426612854, + "learning_rate": 0.0002639447877206115, + "loss": 0.629054069519043, + "mean_token_accuracy": 0.8183021235466004, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5833694102223387, + "eval_loss": 0.6328718662261963, + "eval_mean_token_accuracy": 0.8159044071571114, + "eval_num_tokens": 1110986.0, + "eval_runtime": 161.885, + "eval_samples_per_second": 9.544, + "eval_steps_per_second": 1.198, + "step": 778 + }, + { + "entropy": 0.647241060480927, + "epoch": 2.056628056628057, + "grad_norm": 0.7070767879486084, + "learning_rate": 0.00026188653280135975, + "loss": 0.5823922348022461, + "mean_token_accuracy": 0.8265365301960647, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5995378407835961, + "epoch": 2.1853281853281854, + "grad_norm": 0.8090486526489258, + "learning_rate": 0.0002595738497351955, + "loss": 0.5325597763061524, + "mean_token_accuracy": 0.8369336777925491, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.6043145382404327, + "epoch": 2.314028314028314, + "grad_norm": 0.8279913067817688, + "learning_rate": 0.00025701139329823054, + "loss": 0.5414446258544922, + "mean_token_accuracy": 0.8361396533250809, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5953224584460258, + "epoch": 2.4427284427284426, + "grad_norm": 0.6075023412704468, + "learning_rate": 0.00025420432098964183, + "loss": 0.536654167175293, + "mean_token_accuracy": 0.8340093129873276, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5998479858040809, + "epoch": 2.571428571428571, + "grad_norm": 1.0311471223831177, + "learning_rate": 0.0002511582826510862, + "loss": 0.5372924423217773, + "mean_token_accuracy": 0.8366045409440994, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5980158120393753, + "epoch": 2.7001287001287, + "grad_norm": 0.5971426367759705, + "learning_rate": 0.0002478794090951689, + "loss": 0.5392885208129883, + "mean_token_accuracy": 0.8347727072238922, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.5990680930018425, + "epoch": 2.828828828828829, + "grad_norm": 0.5662627220153809, + "learning_rate": 0.0002443742997658538, + "loss": 0.5360498428344727, + "mean_token_accuracy": 0.8371847170591354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5957039377093315, + "epoch": 2.9575289575289574, + "grad_norm": 0.5043798685073853, + "learning_rate": 0.00024065000945565205, + "loss": 0.5342231369018555, + "mean_token_accuracy": 0.8380735236406326, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5346978819861854, + "eval_loss": 0.6255015134811401, + "eval_mean_token_accuracy": 0.8202014476368108, + "eval_num_tokens": 1666479.0, + "eval_runtime": 161.6098, + "eval_samples_per_second": 9.56, + "eval_steps_per_second": 1.2, + "step": 1167 + }, + { + "entropy": 0.5212209137401196, + "epoch": 3.0849420849420848, + "grad_norm": 0.7095440626144409, + "learning_rate": 0.00023671403410632178, + "loss": 0.45311901092529294, + "mean_token_accuracy": 0.856362871449403, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.4701593083143234, + "epoch": 3.213642213642214, + "grad_norm": 0.6408083438873291, + "learning_rate": 0.0002325742957216607, + "loss": 0.39916397094726563, + "mean_token_accuracy": 0.8698061722517013, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4766591975092888, + "epoch": 3.3423423423423424, + "grad_norm": 0.6415093541145325, + "learning_rate": 0.0002282391264227552, + "loss": 0.4116698455810547, + "mean_token_accuracy": 0.8679435575008392, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.4937947469949722, + "epoch": 3.471042471042471, + "grad_norm": 0.6549825072288513, + "learning_rate": 0.00022371725167778054, + "loss": 0.4296376037597656, + "mean_token_accuracy": 0.8609357953071595, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.4920153194665909, + "epoch": 3.5997425997425996, + "grad_norm": 0.6452126502990723, + "learning_rate": 0.00021901777274010406, + "loss": 0.4307489013671875, + "mean_token_accuracy": 0.8606827831268311, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.490042342543602, + "epoch": 3.7284427284427286, + "grad_norm": 0.5727734565734863, + "learning_rate": 0.0002141501483300395, + "loss": 0.4295254135131836, + "mean_token_accuracy": 0.8616545403003693, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.49857193052768706, + "epoch": 3.857142857142857, + "grad_norm": 0.7732954025268555, + "learning_rate": 0.00020912417559712133, + "loss": 0.4289303207397461, + "mean_token_accuracy": 0.8616443765163422, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.4746784272789955, + "epoch": 3.985842985842986, + "grad_norm": 0.5791187882423401, + "learning_rate": 0.00020394997040121726, + "loss": 0.4180263900756836, + "mean_token_accuracy": 0.866080379486084, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4793926059585257, + "eval_loss": 0.627627968788147, + "eval_mean_token_accuracy": 0.8233870095813397, + "eval_num_tokens": 2221972.0, + "eval_runtime": 162.0225, + "eval_samples_per_second": 9.536, + "eval_steps_per_second": 1.197, + "step": 1556 + }, + { + "entropy": 0.38195489000792454, + "epoch": 4.113256113256114, + "grad_norm": 0.6002617478370667, + "learning_rate": 0.0001986379469521669, + "loss": 0.30819049835205076, + "mean_token_accuracy": 0.8977848634575353, + "num_tokens": 2282164.0, + "step": 1600 + }, + { + "entropy": 0.3655787402391434, + "epoch": 4.241956241956242, + "grad_norm": 0.7100041508674622, + "learning_rate": 0.00019319879684892634, + "loss": 0.29959835052490236, + "mean_token_accuracy": 0.8991208010911942, + "num_tokens": 2353213.0, + "step": 1650 + }, + { + "entropy": 0.3821141055226326, + "epoch": 4.370656370656371, + "grad_norm": 0.5848307013511658, + "learning_rate": 0.00018764346756040715, + "loss": 0.313802490234375, + "mean_token_accuracy": 0.895167955160141, + "num_tokens": 2425068.0, + "step": 1700 + }, + { + "entropy": 0.37083797007799146, + "epoch": 4.499356499356499, + "grad_norm": 0.6447024941444397, + "learning_rate": 0.00018198314039132143, + "loss": 0.30583988189697264, + "mean_token_accuracy": 0.8961733293533325, + "num_tokens": 2498321.0, + "step": 1750 + }, + { + "entropy": 0.3791545969247818, + "epoch": 4.628056628056628, + "grad_norm": 0.6575382351875305, + "learning_rate": 0.00017622920797738184, + "loss": 0.3088031005859375, + "mean_token_accuracy": 0.8960050916671753, + "num_tokens": 2570321.0, + "step": 1800 + }, + { + "entropy": 0.3946831756830215, + "epoch": 4.756756756756757, + "grad_norm": 0.5351552963256836, + "learning_rate": 0.00017039325135515207, + "loss": 0.3229162979125977, + "mean_token_accuracy": 0.8920552498102188, + "num_tokens": 2642851.0, + "step": 1850 + }, + { + "entropy": 0.37298239797353744, + "epoch": 4.885456885456885, + "grad_norm": 0.7624587416648865, + "learning_rate": 0.00016448701665269964, + "loss": 0.3067934799194336, + "mean_token_accuracy": 0.8951873427629471, + "num_tokens": 2715629.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3738150204887095, + "eval_loss": 0.7131896615028381, + "eval_mean_token_accuracy": 0.8207752468045225, + "eval_num_tokens": 2777465.0, + "eval_runtime": 162.1244, + "eval_samples_per_second": 9.53, + "eval_steps_per_second": 1.197, + "step": 1945 + }, + { + "entropy": 0.377820266617669, + "epoch": 5.012870012870013, + "grad_norm": 0.41990190744400024, + "learning_rate": 0.00015852239144796624, + "loss": 0.3058685111999512, + "mean_token_accuracy": 0.8964343480389527, + "num_tokens": 2784896.0, + "step": 1950 + }, + { + "entropy": 0.2714502356946468, + "epoch": 5.141570141570142, + "grad_norm": 0.422568678855896, + "learning_rate": 0.00015251138084243995, + "loss": 0.2093442153930664, + "mean_token_accuracy": 0.9311346983909607, + "num_tokens": 2854374.0, + "step": 2000 + }, + { + "entropy": 0.268475965410471, + "epoch": 5.27027027027027, + "grad_norm": 0.6637414693832397, + "learning_rate": 0.0001464660832982852, + "loss": 0.20736080169677734, + "mean_token_accuracy": 0.9289199805259705, + "num_tokens": 2927362.0, + "step": 2050 + }, + { + "entropy": 0.2644876340031624, + "epoch": 5.398970398970399, + "grad_norm": 0.47317707538604736, + "learning_rate": 0.00014039866628756467, + "loss": 0.20464908599853515, + "mean_token_accuracy": 0.9300856202840805, + "num_tokens": 3000143.0, + "step": 2100 + }, + { + "entropy": 0.2675253136456013, + "epoch": 5.527670527670527, + "grad_norm": 0.5253982543945312, + "learning_rate": 0.00013432134180256338, + "loss": 0.21154335021972656, + "mean_token_accuracy": 0.9283734840154648, + "num_tokens": 3072561.0, + "step": 2150 + }, + { + "entropy": 0.27213907435536383, + "epoch": 5.656370656370656, + "grad_norm": 0.46738553047180176, + "learning_rate": 0.00012824634177650664, + "loss": 0.21339216232299804, + "mean_token_accuracy": 0.9272083270549775, + "num_tokens": 3144831.0, + "step": 2200 + }, + { + "entropy": 0.2785488124191761, + "epoch": 5.785070785070785, + "grad_norm": 0.4469502866268158, + "learning_rate": 0.00012218589346414205, + "loss": 0.21601097106933595, + "mean_token_accuracy": 0.9255663657188415, + "num_tokens": 3215960.0, + "step": 2250 + }, + { + "entropy": 0.2699935150146484, + "epoch": 5.913770913770914, + "grad_norm": 0.7359778881072998, + "learning_rate": 0.00011615219483173828, + "loss": 0.20725584030151367, + "mean_token_accuracy": 0.9286630594730377, + "num_tokens": 3287499.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.26417383682174783, + "eval_loss": 0.8880229592323303, + "eval_mean_token_accuracy": 0.8159987201395723, + "eval_num_tokens": 3332958.0, + "eval_runtime": 162.0991, + "eval_samples_per_second": 9.531, + "eval_steps_per_second": 1.197, + "step": 2334 + }, + { + "entropy": 0.24814540704693458, + "epoch": 6.041184041184041, + "grad_norm": 0.4953760802745819, + "learning_rate": 0.00011015739000603316, + "loss": 0.18749794006347656, + "mean_token_accuracy": 0.9370789509831052, + "num_tokens": 3356879.0, + "step": 2350 + }, + { + "entropy": 0.19976271741092205, + "epoch": 6.1698841698841695, + "grad_norm": 0.4834803342819214, + "learning_rate": 0.00010421354483154553, + "loss": 0.14283526420593262, + "mean_token_accuracy": 0.9521516615152359, + "num_tokens": 3427587.0, + "step": 2400 + }, + { + "entropy": 0.2060488449037075, + "epoch": 6.298584298584299, + "grad_norm": 0.4888673722743988, + "learning_rate": 9.8332622585447e-05, + "loss": 0.14414511680603026, + "mean_token_accuracy": 0.9510996866226197, + "num_tokens": 3498688.0, + "step": 2450 + }, + { + "entropy": 0.2059111550450325, + "epoch": 6.427284427284428, + "grad_norm": 0.4064404368400574, + "learning_rate": 9.252645989887253e-05, + "loss": 0.14820143699645996, + "mean_token_accuracy": 0.9507584601640702, + "num_tokens": 3566137.0, + "step": 2500 + }, + { + "entropy": 0.19700154662132263, + "epoch": 6.555984555984556, + "grad_norm": 0.467965304851532, + "learning_rate": 8.680674293313417e-05, + "loss": 0.14303470611572267, + "mean_token_accuracy": 0.9515972435474396, + "num_tokens": 3639573.0, + "step": 2550 + }, + { + "entropy": 0.20180423602461814, + "epoch": 6.684684684684685, + "grad_norm": 0.36836138367652893, + "learning_rate": 8.118498385878736e-05, + "loss": 0.14280882835388184, + "mean_token_accuracy": 0.9515993863344192, + "num_tokens": 3710433.0, + "step": 2600 + }, + { + "entropy": 0.20024395987391472, + "epoch": 6.813384813384813, + "grad_norm": 0.38375866413116455, + "learning_rate": 7.567249768489171e-05, + "loss": 0.1427844524383545, + "mean_token_accuracy": 0.9524166631698608, + "num_tokens": 3781550.0, + "step": 2650 + }, + { + "entropy": 0.19561587080359458, + "epoch": 6.942084942084942, + "grad_norm": 0.41185441613197327, + "learning_rate": 7.028037948510187e-05, + "loss": 0.13993803024291993, + "mean_token_accuracy": 0.9522478264570237, + "num_tokens": 3854952.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.19501976062034823, + "eval_loss": 1.0653952360153198, + "eval_mean_token_accuracy": 0.8205490803595671, + "eval_num_tokens": 3888451.0, + "eval_runtime": 161.8533, + "eval_samples_per_second": 9.546, + "eval_steps_per_second": 1.199, + "step": 2723 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.516671077296845e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cd0d97a8c611bcad7d4624b6232e1767e61fe6d3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.00237968804112545, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b9cc3bfc29cf95d841f55542052c625511ea95a7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/trainer_state.json @@ -0,0 +1,742 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3112, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6110110306739807, + "epoch": 0.1287001287001287, + "grad_norm": 0.8611119389533997, + "learning_rate": 3.4130257962133866e-05, + "loss": 1.53956298828125, + "mean_token_accuracy": 0.6652342769503593, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.8662820833921433, + "epoch": 0.2574002574002574, + "grad_norm": 0.7405035495758057, + "learning_rate": 6.895705180104598e-05, + "loss": 0.7929539489746094, + "mean_token_accuracy": 0.7786347842216492, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7912819278240204, + "epoch": 0.3861003861003861, + "grad_norm": 0.6105485558509827, + "learning_rate": 0.00010378384563995809, + "loss": 0.7236511993408203, + "mean_token_accuracy": 0.7925467795133591, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7777709531784057, + "epoch": 0.5148005148005148, + "grad_norm": 0.5781793594360352, + "learning_rate": 0.0001386106394788702, + "loss": 0.7024919891357422, + "mean_token_accuracy": 0.7969876372814179, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7632609683275223, + "epoch": 0.6435006435006435, + "grad_norm": 0.6553444862365723, + "learning_rate": 0.00017343743331778232, + "loss": 0.7000718688964844, + "mean_token_accuracy": 0.7994742071628571, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.754165632724762, + "epoch": 0.7722007722007722, + "grad_norm": 0.634222149848938, + "learning_rate": 0.00020826422715669444, + "loss": 0.6909049987792969, + "mean_token_accuracy": 0.8016809666156769, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.7530711203813553, + "epoch": 0.9009009009009009, + "grad_norm": 0.7020156383514404, + "learning_rate": 0.00024309102099560653, + "loss": 0.69554931640625, + "mean_token_accuracy": 0.8002807641029358, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.5728044164242204, + "eval_loss": 0.6785851716995239, + "eval_mean_token_accuracy": 0.798433246993527, + "eval_num_tokens": 555493.0, + "eval_runtime": 162.4129, + "eval_samples_per_second": 9.513, + "eval_steps_per_second": 1.194, + "step": 389 + }, + { + "entropy": 0.7489313946829902, + "epoch": 1.0283140283140284, + "grad_norm": 0.7532815933227539, + "learning_rate": 0.0002709470016827303, + "loss": 0.6856581878662109, + "mean_token_accuracy": 0.8023027079273956, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7238409864902496, + "epoch": 1.157014157014157, + "grad_norm": 1.2016215324401855, + "learning_rate": 0.0002707561443541359, + "loss": 0.6699818420410156, + "mean_token_accuracy": 0.8070302194356919, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7388938587903976, + "epoch": 1.2857142857142856, + "grad_norm": 0.7279272079467773, + "learning_rate": 0.0002702930068622498, + "loss": 0.6728517150878907, + "mean_token_accuracy": 0.8049580943584442, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7284485149383545, + "epoch": 1.4144144144144144, + "grad_norm": 0.8563987016677856, + "learning_rate": 0.0002695585213716931, + "loss": 0.6657986450195312, + "mean_token_accuracy": 0.8085588800907135, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6960959500074386, + "epoch": 1.5431145431145432, + "grad_norm": 0.5145474672317505, + "learning_rate": 0.0002685541661937683, + "loss": 0.6358638763427734, + "mean_token_accuracy": 0.8131621342897415, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.7220414417982102, + "epoch": 1.6718146718146718, + "grad_norm": 0.9381059408187866, + "learning_rate": 0.00026728196281103746, + "loss": 0.6531407928466797, + "mean_token_accuracy": 0.811619822382927, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6955459499359131, + "epoch": 1.8005148005148004, + "grad_norm": 0.6115108728408813, + "learning_rate": 0.0002657444718086503, + "loss": 0.6269588088989257, + "mean_token_accuracy": 0.8155373805761337, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6929812705516816, + "epoch": 1.9292149292149292, + "grad_norm": 0.749489426612854, + "learning_rate": 0.0002639447877206115, + "loss": 0.629054069519043, + "mean_token_accuracy": 0.8183021235466004, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5833694102223387, + "eval_loss": 0.6328718662261963, + "eval_mean_token_accuracy": 0.8159044071571114, + "eval_num_tokens": 1110986.0, + "eval_runtime": 161.885, + "eval_samples_per_second": 9.544, + "eval_steps_per_second": 1.198, + "step": 778 + }, + { + "entropy": 0.647241060480927, + "epoch": 2.056628056628057, + "grad_norm": 0.7070767879486084, + "learning_rate": 0.00026188653280135975, + "loss": 0.5823922348022461, + "mean_token_accuracy": 0.8265365301960647, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5995378407835961, + "epoch": 2.1853281853281854, + "grad_norm": 0.8090486526489258, + "learning_rate": 0.0002595738497351955, + "loss": 0.5325597763061524, + "mean_token_accuracy": 0.8369336777925491, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.6043145382404327, + "epoch": 2.314028314028314, + "grad_norm": 0.8279913067817688, + "learning_rate": 0.00025701139329823054, + "loss": 0.5414446258544922, + "mean_token_accuracy": 0.8361396533250809, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5953224584460258, + "epoch": 2.4427284427284426, + "grad_norm": 0.6075023412704468, + "learning_rate": 0.00025420432098964183, + "loss": 0.536654167175293, + "mean_token_accuracy": 0.8340093129873276, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5998479858040809, + "epoch": 2.571428571428571, + "grad_norm": 1.0311471223831177, + "learning_rate": 0.0002511582826510862, + "loss": 0.5372924423217773, + "mean_token_accuracy": 0.8366045409440994, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5980158120393753, + "epoch": 2.7001287001287, + "grad_norm": 0.5971426367759705, + "learning_rate": 0.0002478794090951689, + "loss": 0.5392885208129883, + "mean_token_accuracy": 0.8347727072238922, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.5990680930018425, + "epoch": 2.828828828828829, + "grad_norm": 0.5662627220153809, + "learning_rate": 0.0002443742997658538, + "loss": 0.5360498428344727, + "mean_token_accuracy": 0.8371847170591354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5957039377093315, + "epoch": 2.9575289575289574, + "grad_norm": 0.5043798685073853, + "learning_rate": 0.00024065000945565205, + "loss": 0.5342231369018555, + "mean_token_accuracy": 0.8380735236406326, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5346978819861854, + "eval_loss": 0.6255015134811401, + "eval_mean_token_accuracy": 0.8202014476368108, + "eval_num_tokens": 1666479.0, + "eval_runtime": 161.6098, + "eval_samples_per_second": 9.56, + "eval_steps_per_second": 1.2, + "step": 1167 + }, + { + "entropy": 0.5212209137401196, + "epoch": 3.0849420849420848, + "grad_norm": 0.7095440626144409, + "learning_rate": 0.00023671403410632178, + "loss": 0.45311901092529294, + "mean_token_accuracy": 0.856362871449403, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.4701593083143234, + "epoch": 3.213642213642214, + "grad_norm": 0.6408083438873291, + "learning_rate": 0.0002325742957216607, + "loss": 0.39916397094726563, + "mean_token_accuracy": 0.8698061722517013, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4766591975092888, + "epoch": 3.3423423423423424, + "grad_norm": 0.6415093541145325, + "learning_rate": 0.0002282391264227552, + "loss": 0.4116698455810547, + "mean_token_accuracy": 0.8679435575008392, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.4937947469949722, + "epoch": 3.471042471042471, + "grad_norm": 0.6549825072288513, + "learning_rate": 0.00022371725167778054, + "loss": 0.4296376037597656, + "mean_token_accuracy": 0.8609357953071595, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.4920153194665909, + "epoch": 3.5997425997425996, + "grad_norm": 0.6452126502990723, + "learning_rate": 0.00021901777274010406, + "loss": 0.4307489013671875, + "mean_token_accuracy": 0.8606827831268311, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.490042342543602, + "epoch": 3.7284427284427286, + "grad_norm": 0.5727734565734863, + "learning_rate": 0.0002141501483300395, + "loss": 0.4295254135131836, + "mean_token_accuracy": 0.8616545403003693, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.49857193052768706, + "epoch": 3.857142857142857, + "grad_norm": 0.7732954025268555, + "learning_rate": 0.00020912417559712133, + "loss": 0.4289303207397461, + "mean_token_accuracy": 0.8616443765163422, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.4746784272789955, + "epoch": 3.985842985842986, + "grad_norm": 0.5791187882423401, + "learning_rate": 0.00020394997040121726, + "loss": 0.4180263900756836, + "mean_token_accuracy": 0.866080379486084, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4793926059585257, + "eval_loss": 0.627627968788147, + "eval_mean_token_accuracy": 0.8233870095813397, + "eval_num_tokens": 2221972.0, + "eval_runtime": 162.0225, + "eval_samples_per_second": 9.536, + "eval_steps_per_second": 1.197, + "step": 1556 + }, + { + "entropy": 0.38195489000792454, + "epoch": 4.113256113256114, + "grad_norm": 0.6002617478370667, + "learning_rate": 0.0001986379469521669, + "loss": 0.30819049835205076, + "mean_token_accuracy": 0.8977848634575353, + "num_tokens": 2282164.0, + "step": 1600 + }, + { + "entropy": 0.3655787402391434, + "epoch": 4.241956241956242, + "grad_norm": 0.7100041508674622, + "learning_rate": 0.00019319879684892634, + "loss": 0.29959835052490236, + "mean_token_accuracy": 0.8991208010911942, + "num_tokens": 2353213.0, + "step": 1650 + }, + { + "entropy": 0.3821141055226326, + "epoch": 4.370656370656371, + "grad_norm": 0.5848307013511658, + "learning_rate": 0.00018764346756040715, + "loss": 0.313802490234375, + "mean_token_accuracy": 0.895167955160141, + "num_tokens": 2425068.0, + "step": 1700 + }, + { + "entropy": 0.37083797007799146, + "epoch": 4.499356499356499, + "grad_norm": 0.6447024941444397, + "learning_rate": 0.00018198314039132143, + "loss": 0.30583988189697264, + "mean_token_accuracy": 0.8961733293533325, + "num_tokens": 2498321.0, + "step": 1750 + }, + { + "entropy": 0.3791545969247818, + "epoch": 4.628056628056628, + "grad_norm": 0.6575382351875305, + "learning_rate": 0.00017622920797738184, + "loss": 0.3088031005859375, + "mean_token_accuracy": 0.8960050916671753, + "num_tokens": 2570321.0, + "step": 1800 + }, + { + "entropy": 0.3946831756830215, + "epoch": 4.756756756756757, + "grad_norm": 0.5351552963256836, + "learning_rate": 0.00017039325135515207, + "loss": 0.3229162979125977, + "mean_token_accuracy": 0.8920552498102188, + "num_tokens": 2642851.0, + "step": 1850 + }, + { + "entropy": 0.37298239797353744, + "epoch": 4.885456885456885, + "grad_norm": 0.7624587416648865, + "learning_rate": 0.00016448701665269964, + "loss": 0.3067934799194336, + "mean_token_accuracy": 0.8951873427629471, + "num_tokens": 2715629.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3738150204887095, + "eval_loss": 0.7131896615028381, + "eval_mean_token_accuracy": 0.8207752468045225, + "eval_num_tokens": 2777465.0, + "eval_runtime": 162.1244, + "eval_samples_per_second": 9.53, + "eval_steps_per_second": 1.197, + "step": 1945 + }, + { + "entropy": 0.377820266617669, + "epoch": 5.012870012870013, + "grad_norm": 0.41990190744400024, + "learning_rate": 0.00015852239144796624, + "loss": 0.3058685111999512, + "mean_token_accuracy": 0.8964343480389527, + "num_tokens": 2784896.0, + "step": 1950 + }, + { + "entropy": 0.2714502356946468, + "epoch": 5.141570141570142, + "grad_norm": 0.422568678855896, + "learning_rate": 0.00015251138084243995, + "loss": 0.2093442153930664, + "mean_token_accuracy": 0.9311346983909607, + "num_tokens": 2854374.0, + "step": 2000 + }, + { + "entropy": 0.268475965410471, + "epoch": 5.27027027027027, + "grad_norm": 0.6637414693832397, + "learning_rate": 0.0001464660832982852, + "loss": 0.20736080169677734, + "mean_token_accuracy": 0.9289199805259705, + "num_tokens": 2927362.0, + "step": 2050 + }, + { + "entropy": 0.2644876340031624, + "epoch": 5.398970398970399, + "grad_norm": 0.47317707538604736, + "learning_rate": 0.00014039866628756467, + "loss": 0.20464908599853515, + "mean_token_accuracy": 0.9300856202840805, + "num_tokens": 3000143.0, + "step": 2100 + }, + { + "entropy": 0.2675253136456013, + "epoch": 5.527670527670527, + "grad_norm": 0.5253982543945312, + "learning_rate": 0.00013432134180256338, + "loss": 0.21154335021972656, + "mean_token_accuracy": 0.9283734840154648, + "num_tokens": 3072561.0, + "step": 2150 + }, + { + "entropy": 0.27213907435536383, + "epoch": 5.656370656370656, + "grad_norm": 0.46738553047180176, + "learning_rate": 0.00012824634177650664, + "loss": 0.21339216232299804, + "mean_token_accuracy": 0.9272083270549775, + "num_tokens": 3144831.0, + "step": 2200 + }, + { + "entropy": 0.2785488124191761, + "epoch": 5.785070785070785, + "grad_norm": 0.4469502866268158, + "learning_rate": 0.00012218589346414205, + "loss": 0.21601097106933595, + "mean_token_accuracy": 0.9255663657188415, + "num_tokens": 3215960.0, + "step": 2250 + }, + { + "entropy": 0.2699935150146484, + "epoch": 5.913770913770914, + "grad_norm": 0.7359778881072998, + "learning_rate": 0.00011615219483173828, + "loss": 0.20725584030151367, + "mean_token_accuracy": 0.9286630594730377, + "num_tokens": 3287499.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.26417383682174783, + "eval_loss": 0.8880229592323303, + "eval_mean_token_accuracy": 0.8159987201395723, + "eval_num_tokens": 3332958.0, + "eval_runtime": 162.0991, + "eval_samples_per_second": 9.531, + "eval_steps_per_second": 1.197, + "step": 2334 + }, + { + "entropy": 0.24814540704693458, + "epoch": 6.041184041184041, + "grad_norm": 0.4953760802745819, + "learning_rate": 0.00011015739000603316, + "loss": 0.18749794006347656, + "mean_token_accuracy": 0.9370789509831052, + "num_tokens": 3356879.0, + "step": 2350 + }, + { + "entropy": 0.19976271741092205, + "epoch": 6.1698841698841695, + "grad_norm": 0.4834803342819214, + "learning_rate": 0.00010421354483154553, + "loss": 0.14283526420593262, + "mean_token_accuracy": 0.9521516615152359, + "num_tokens": 3427587.0, + "step": 2400 + }, + { + "entropy": 0.2060488449037075, + "epoch": 6.298584298584299, + "grad_norm": 0.4888673722743988, + "learning_rate": 9.8332622585447e-05, + "loss": 0.14414511680603026, + "mean_token_accuracy": 0.9510996866226197, + "num_tokens": 3498688.0, + "step": 2450 + }, + { + "entropy": 0.2059111550450325, + "epoch": 6.427284427284428, + "grad_norm": 0.4064404368400574, + "learning_rate": 9.252645989887253e-05, + "loss": 0.14820143699645996, + "mean_token_accuracy": 0.9507584601640702, + "num_tokens": 3566137.0, + "step": 2500 + }, + { + "entropy": 0.19700154662132263, + "epoch": 6.555984555984556, + "grad_norm": 0.467965304851532, + "learning_rate": 8.680674293313417e-05, + "loss": 0.14303470611572267, + "mean_token_accuracy": 0.9515972435474396, + "num_tokens": 3639573.0, + "step": 2550 + }, + { + "entropy": 0.20180423602461814, + "epoch": 6.684684684684685, + "grad_norm": 0.36836138367652893, + "learning_rate": 8.118498385878736e-05, + "loss": 0.14280882835388184, + "mean_token_accuracy": 0.9515993863344192, + "num_tokens": 3710433.0, + "step": 2600 + }, + { + "entropy": 0.20024395987391472, + "epoch": 6.813384813384813, + "grad_norm": 0.38375866413116455, + "learning_rate": 7.567249768489171e-05, + "loss": 0.1427844524383545, + "mean_token_accuracy": 0.9524166631698608, + "num_tokens": 3781550.0, + "step": 2650 + }, + { + "entropy": 0.19561587080359458, + "epoch": 6.942084942084942, + "grad_norm": 0.41185441613197327, + "learning_rate": 7.028037948510187e-05, + "loss": 0.13993803024291993, + "mean_token_accuracy": 0.9522478264570237, + "num_tokens": 3854952.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.19501976062034823, + "eval_loss": 1.0653952360153198, + "eval_mean_token_accuracy": 0.8205490803595671, + "eval_num_tokens": 3888451.0, + "eval_runtime": 161.8533, + "eval_samples_per_second": 9.546, + "eval_steps_per_second": 1.199, + "step": 2723 + }, + { + "entropy": 0.17789882526855277, + "epoch": 7.06949806949807, + "grad_norm": 0.41413992643356323, + "learning_rate": 6.50194820664261e-05, + "loss": 0.12078390121459961, + "mean_token_accuracy": 0.9589925727458916, + "num_tokens": 3928354.0, + "step": 2750 + }, + { + "entropy": 0.16781829454004765, + "epoch": 7.198198198198198, + "grad_norm": 0.25806066393852234, + "learning_rate": 5.990039412559906e-05, + "loss": 0.10963023185729981, + "mean_token_accuracy": 0.9617267113924026, + "num_tokens": 4000113.0, + "step": 2800 + }, + { + "entropy": 0.1649068508297205, + "epoch": 7.326898326898327, + "grad_norm": 0.27411890029907227, + "learning_rate": 5.493341893703393e-05, + "loss": 0.11152458190917969, + "mean_token_accuracy": 0.9620639663934708, + "num_tokens": 4071032.0, + "step": 2850 + }, + { + "entropy": 0.161333369910717, + "epoch": 7.455598455598455, + "grad_norm": 0.24944494664669037, + "learning_rate": 5.0128553615248396e-05, + "loss": 0.1094522476196289, + "mean_token_accuracy": 0.962428919672966, + "num_tokens": 4143616.0, + "step": 2900 + }, + { + "entropy": 0.15613057143986225, + "epoch": 7.584298584298584, + "grad_norm": 0.1455036848783493, + "learning_rate": 4.549546899350423e-05, + "loss": 0.11092090606689453, + "mean_token_accuracy": 0.9620462411642074, + "num_tokens": 4215664.0, + "step": 2950 + }, + { + "entropy": 0.1631234459578991, + "epoch": 7.712998712998713, + "grad_norm": 0.2129560261964798, + "learning_rate": 4.104349015915862e-05, + "loss": 0.1141857624053955, + "mean_token_accuracy": 0.9613765001296997, + "num_tokens": 4286387.0, + "step": 3000 + }, + { + "entropy": 0.1680422095954418, + "epoch": 7.841698841698841, + "grad_norm": 0.24886097013950348, + "learning_rate": 3.678157768490372e-05, + "loss": 0.11513191223144531, + "mean_token_accuracy": 0.9615794748067856, + "num_tokens": 4355875.0, + "step": 3050 + }, + { + "entropy": 0.16354035697877406, + "epoch": 7.97039897039897, + "grad_norm": 0.27600204944610596, + "learning_rate": 3.27183095936714e-05, + "loss": 0.1118631362915039, + "mean_token_accuracy": 0.9623224419355393, + "num_tokens": 4427088.0, + "step": 3100 + }, + { + "epoch": 8.0, + "eval_entropy": 0.16586771200305409, + "eval_loss": 1.204746961593628, + "eval_mean_token_accuracy": 0.8229975042883882, + "eval_num_tokens": 4443944.0, + "eval_runtime": 162.0251, + "eval_samples_per_second": 9.536, + "eval_steps_per_second": 1.197, + "step": 3112 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.445001770940518e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cd0d97a8c611bcad7d4624b6232e1767e61fe6d3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.00237968804112545, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ceadb30374b4b3a3bd1056672155d8b2d2878133 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/trainer_state.json @@ -0,0 +1,833 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3501, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6110110306739807, + "epoch": 0.1287001287001287, + "grad_norm": 0.8611119389533997, + "learning_rate": 3.4130257962133866e-05, + "loss": 1.53956298828125, + "mean_token_accuracy": 0.6652342769503593, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.8662820833921433, + "epoch": 0.2574002574002574, + "grad_norm": 0.7405035495758057, + "learning_rate": 6.895705180104598e-05, + "loss": 0.7929539489746094, + "mean_token_accuracy": 0.7786347842216492, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7912819278240204, + "epoch": 0.3861003861003861, + "grad_norm": 0.6105485558509827, + "learning_rate": 0.00010378384563995809, + "loss": 0.7236511993408203, + "mean_token_accuracy": 0.7925467795133591, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7777709531784057, + "epoch": 0.5148005148005148, + "grad_norm": 0.5781793594360352, + "learning_rate": 0.0001386106394788702, + "loss": 0.7024919891357422, + "mean_token_accuracy": 0.7969876372814179, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7632609683275223, + "epoch": 0.6435006435006435, + "grad_norm": 0.6553444862365723, + "learning_rate": 0.00017343743331778232, + "loss": 0.7000718688964844, + "mean_token_accuracy": 0.7994742071628571, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.754165632724762, + "epoch": 0.7722007722007722, + "grad_norm": 0.634222149848938, + "learning_rate": 0.00020826422715669444, + "loss": 0.6909049987792969, + "mean_token_accuracy": 0.8016809666156769, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.7530711203813553, + "epoch": 0.9009009009009009, + "grad_norm": 0.7020156383514404, + "learning_rate": 0.00024309102099560653, + "loss": 0.69554931640625, + "mean_token_accuracy": 0.8002807641029358, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.5728044164242204, + "eval_loss": 0.6785851716995239, + "eval_mean_token_accuracy": 0.798433246993527, + "eval_num_tokens": 555493.0, + "eval_runtime": 162.4129, + "eval_samples_per_second": 9.513, + "eval_steps_per_second": 1.194, + "step": 389 + }, + { + "entropy": 0.7489313946829902, + "epoch": 1.0283140283140284, + "grad_norm": 0.7532815933227539, + "learning_rate": 0.0002709470016827303, + "loss": 0.6856581878662109, + "mean_token_accuracy": 0.8023027079273956, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7238409864902496, + "epoch": 1.157014157014157, + "grad_norm": 1.2016215324401855, + "learning_rate": 0.0002707561443541359, + "loss": 0.6699818420410156, + "mean_token_accuracy": 0.8070302194356919, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7388938587903976, + "epoch": 1.2857142857142856, + "grad_norm": 0.7279272079467773, + "learning_rate": 0.0002702930068622498, + "loss": 0.6728517150878907, + "mean_token_accuracy": 0.8049580943584442, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7284485149383545, + "epoch": 1.4144144144144144, + "grad_norm": 0.8563987016677856, + "learning_rate": 0.0002695585213716931, + "loss": 0.6657986450195312, + "mean_token_accuracy": 0.8085588800907135, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6960959500074386, + "epoch": 1.5431145431145432, + "grad_norm": 0.5145474672317505, + "learning_rate": 0.0002685541661937683, + "loss": 0.6358638763427734, + "mean_token_accuracy": 0.8131621342897415, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.7220414417982102, + "epoch": 1.6718146718146718, + "grad_norm": 0.9381059408187866, + "learning_rate": 0.00026728196281103746, + "loss": 0.6531407928466797, + "mean_token_accuracy": 0.811619822382927, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6955459499359131, + "epoch": 1.8005148005148004, + "grad_norm": 0.6115108728408813, + "learning_rate": 0.0002657444718086503, + "loss": 0.6269588088989257, + "mean_token_accuracy": 0.8155373805761337, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6929812705516816, + "epoch": 1.9292149292149292, + "grad_norm": 0.749489426612854, + "learning_rate": 0.0002639447877206115, + "loss": 0.629054069519043, + "mean_token_accuracy": 0.8183021235466004, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5833694102223387, + "eval_loss": 0.6328718662261963, + "eval_mean_token_accuracy": 0.8159044071571114, + "eval_num_tokens": 1110986.0, + "eval_runtime": 161.885, + "eval_samples_per_second": 9.544, + "eval_steps_per_second": 1.198, + "step": 778 + }, + { + "entropy": 0.647241060480927, + "epoch": 2.056628056628057, + "grad_norm": 0.7070767879486084, + "learning_rate": 0.00026188653280135975, + "loss": 0.5823922348022461, + "mean_token_accuracy": 0.8265365301960647, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5995378407835961, + "epoch": 2.1853281853281854, + "grad_norm": 0.8090486526489258, + "learning_rate": 0.0002595738497351955, + "loss": 0.5325597763061524, + "mean_token_accuracy": 0.8369336777925491, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.6043145382404327, + "epoch": 2.314028314028314, + "grad_norm": 0.8279913067817688, + "learning_rate": 0.00025701139329823054, + "loss": 0.5414446258544922, + "mean_token_accuracy": 0.8361396533250809, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5953224584460258, + "epoch": 2.4427284427284426, + "grad_norm": 0.6075023412704468, + "learning_rate": 0.00025420432098964183, + "loss": 0.536654167175293, + "mean_token_accuracy": 0.8340093129873276, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5998479858040809, + "epoch": 2.571428571428571, + "grad_norm": 1.0311471223831177, + "learning_rate": 0.0002511582826510862, + "loss": 0.5372924423217773, + "mean_token_accuracy": 0.8366045409440994, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5980158120393753, + "epoch": 2.7001287001287, + "grad_norm": 0.5971426367759705, + "learning_rate": 0.0002478794090951689, + "loss": 0.5392885208129883, + "mean_token_accuracy": 0.8347727072238922, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.5990680930018425, + "epoch": 2.828828828828829, + "grad_norm": 0.5662627220153809, + "learning_rate": 0.0002443742997658538, + "loss": 0.5360498428344727, + "mean_token_accuracy": 0.8371847170591354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5957039377093315, + "epoch": 2.9575289575289574, + "grad_norm": 0.5043798685073853, + "learning_rate": 0.00024065000945565205, + "loss": 0.5342231369018555, + "mean_token_accuracy": 0.8380735236406326, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5346978819861854, + "eval_loss": 0.6255015134811401, + "eval_mean_token_accuracy": 0.8202014476368108, + "eval_num_tokens": 1666479.0, + "eval_runtime": 161.6098, + "eval_samples_per_second": 9.56, + "eval_steps_per_second": 1.2, + "step": 1167 + }, + { + "entropy": 0.5212209137401196, + "epoch": 3.0849420849420848, + "grad_norm": 0.7095440626144409, + "learning_rate": 0.00023671403410632178, + "loss": 0.45311901092529294, + "mean_token_accuracy": 0.856362871449403, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.4701593083143234, + "epoch": 3.213642213642214, + "grad_norm": 0.6408083438873291, + "learning_rate": 0.0002325742957216607, + "loss": 0.39916397094726563, + "mean_token_accuracy": 0.8698061722517013, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4766591975092888, + "epoch": 3.3423423423423424, + "grad_norm": 0.6415093541145325, + "learning_rate": 0.0002282391264227552, + "loss": 0.4116698455810547, + "mean_token_accuracy": 0.8679435575008392, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.4937947469949722, + "epoch": 3.471042471042471, + "grad_norm": 0.6549825072288513, + "learning_rate": 0.00022371725167778054, + "loss": 0.4296376037597656, + "mean_token_accuracy": 0.8609357953071595, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.4920153194665909, + "epoch": 3.5997425997425996, + "grad_norm": 0.6452126502990723, + "learning_rate": 0.00021901777274010406, + "loss": 0.4307489013671875, + "mean_token_accuracy": 0.8606827831268311, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.490042342543602, + "epoch": 3.7284427284427286, + "grad_norm": 0.5727734565734863, + "learning_rate": 0.0002141501483300395, + "loss": 0.4295254135131836, + "mean_token_accuracy": 0.8616545403003693, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.49857193052768706, + "epoch": 3.857142857142857, + "grad_norm": 0.7732954025268555, + "learning_rate": 0.00020912417559712133, + "loss": 0.4289303207397461, + "mean_token_accuracy": 0.8616443765163422, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.4746784272789955, + "epoch": 3.985842985842986, + "grad_norm": 0.5791187882423401, + "learning_rate": 0.00020394997040121726, + "loss": 0.4180263900756836, + "mean_token_accuracy": 0.866080379486084, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4793926059585257, + "eval_loss": 0.627627968788147, + "eval_mean_token_accuracy": 0.8233870095813397, + "eval_num_tokens": 2221972.0, + "eval_runtime": 162.0225, + "eval_samples_per_second": 9.536, + "eval_steps_per_second": 1.197, + "step": 1556 + }, + { + "entropy": 0.38195489000792454, + "epoch": 4.113256113256114, + "grad_norm": 0.6002617478370667, + "learning_rate": 0.0001986379469521669, + "loss": 0.30819049835205076, + "mean_token_accuracy": 0.8977848634575353, + "num_tokens": 2282164.0, + "step": 1600 + }, + { + "entropy": 0.3655787402391434, + "epoch": 4.241956241956242, + "grad_norm": 0.7100041508674622, + "learning_rate": 0.00019319879684892634, + "loss": 0.29959835052490236, + "mean_token_accuracy": 0.8991208010911942, + "num_tokens": 2353213.0, + "step": 1650 + }, + { + "entropy": 0.3821141055226326, + "epoch": 4.370656370656371, + "grad_norm": 0.5848307013511658, + "learning_rate": 0.00018764346756040715, + "loss": 0.313802490234375, + "mean_token_accuracy": 0.895167955160141, + "num_tokens": 2425068.0, + "step": 1700 + }, + { + "entropy": 0.37083797007799146, + "epoch": 4.499356499356499, + "grad_norm": 0.6447024941444397, + "learning_rate": 0.00018198314039132143, + "loss": 0.30583988189697264, + "mean_token_accuracy": 0.8961733293533325, + "num_tokens": 2498321.0, + "step": 1750 + }, + { + "entropy": 0.3791545969247818, + "epoch": 4.628056628056628, + "grad_norm": 0.6575382351875305, + "learning_rate": 0.00017622920797738184, + "loss": 0.3088031005859375, + "mean_token_accuracy": 0.8960050916671753, + "num_tokens": 2570321.0, + "step": 1800 + }, + { + "entropy": 0.3946831756830215, + "epoch": 4.756756756756757, + "grad_norm": 0.5351552963256836, + "learning_rate": 0.00017039325135515207, + "loss": 0.3229162979125977, + "mean_token_accuracy": 0.8920552498102188, + "num_tokens": 2642851.0, + "step": 1850 + }, + { + "entropy": 0.37298239797353744, + "epoch": 4.885456885456885, + "grad_norm": 0.7624587416648865, + "learning_rate": 0.00016448701665269964, + "loss": 0.3067934799194336, + "mean_token_accuracy": 0.8951873427629471, + "num_tokens": 2715629.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3738150204887095, + "eval_loss": 0.7131896615028381, + "eval_mean_token_accuracy": 0.8207752468045225, + "eval_num_tokens": 2777465.0, + "eval_runtime": 162.1244, + "eval_samples_per_second": 9.53, + "eval_steps_per_second": 1.197, + "step": 1945 + }, + { + "entropy": 0.377820266617669, + "epoch": 5.012870012870013, + "grad_norm": 0.41990190744400024, + "learning_rate": 0.00015852239144796624, + "loss": 0.3058685111999512, + "mean_token_accuracy": 0.8964343480389527, + "num_tokens": 2784896.0, + "step": 1950 + }, + { + "entropy": 0.2714502356946468, + "epoch": 5.141570141570142, + "grad_norm": 0.422568678855896, + "learning_rate": 0.00015251138084243995, + "loss": 0.2093442153930664, + "mean_token_accuracy": 0.9311346983909607, + "num_tokens": 2854374.0, + "step": 2000 + }, + { + "entropy": 0.268475965410471, + "epoch": 5.27027027027027, + "grad_norm": 0.6637414693832397, + "learning_rate": 0.0001464660832982852, + "loss": 0.20736080169677734, + "mean_token_accuracy": 0.9289199805259705, + "num_tokens": 2927362.0, + "step": 2050 + }, + { + "entropy": 0.2644876340031624, + "epoch": 5.398970398970399, + "grad_norm": 0.47317707538604736, + "learning_rate": 0.00014039866628756467, + "loss": 0.20464908599853515, + "mean_token_accuracy": 0.9300856202840805, + "num_tokens": 3000143.0, + "step": 2100 + }, + { + "entropy": 0.2675253136456013, + "epoch": 5.527670527670527, + "grad_norm": 0.5253982543945312, + "learning_rate": 0.00013432134180256338, + "loss": 0.21154335021972656, + "mean_token_accuracy": 0.9283734840154648, + "num_tokens": 3072561.0, + "step": 2150 + }, + { + "entropy": 0.27213907435536383, + "epoch": 5.656370656370656, + "grad_norm": 0.46738553047180176, + "learning_rate": 0.00012824634177650664, + "loss": 0.21339216232299804, + "mean_token_accuracy": 0.9272083270549775, + "num_tokens": 3144831.0, + "step": 2200 + }, + { + "entropy": 0.2785488124191761, + "epoch": 5.785070785070785, + "grad_norm": 0.4469502866268158, + "learning_rate": 0.00012218589346414205, + "loss": 0.21601097106933595, + "mean_token_accuracy": 0.9255663657188415, + "num_tokens": 3215960.0, + "step": 2250 + }, + { + "entropy": 0.2699935150146484, + "epoch": 5.913770913770914, + "grad_norm": 0.7359778881072998, + "learning_rate": 0.00011615219483173828, + "loss": 0.20725584030151367, + "mean_token_accuracy": 0.9286630594730377, + "num_tokens": 3287499.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.26417383682174783, + "eval_loss": 0.8880229592323303, + "eval_mean_token_accuracy": 0.8159987201395723, + "eval_num_tokens": 3332958.0, + "eval_runtime": 162.0991, + "eval_samples_per_second": 9.531, + "eval_steps_per_second": 1.197, + "step": 2334 + }, + { + "entropy": 0.24814540704693458, + "epoch": 6.041184041184041, + "grad_norm": 0.4953760802745819, + "learning_rate": 0.00011015739000603316, + "loss": 0.18749794006347656, + "mean_token_accuracy": 0.9370789509831052, + "num_tokens": 3356879.0, + "step": 2350 + }, + { + "entropy": 0.19976271741092205, + "epoch": 6.1698841698841695, + "grad_norm": 0.4834803342819214, + "learning_rate": 0.00010421354483154553, + "loss": 0.14283526420593262, + "mean_token_accuracy": 0.9521516615152359, + "num_tokens": 3427587.0, + "step": 2400 + }, + { + "entropy": 0.2060488449037075, + "epoch": 6.298584298584299, + "grad_norm": 0.4888673722743988, + "learning_rate": 9.8332622585447e-05, + "loss": 0.14414511680603026, + "mean_token_accuracy": 0.9510996866226197, + "num_tokens": 3498688.0, + "step": 2450 + }, + { + "entropy": 0.2059111550450325, + "epoch": 6.427284427284428, + "grad_norm": 0.4064404368400574, + "learning_rate": 9.252645989887253e-05, + "loss": 0.14820143699645996, + "mean_token_accuracy": 0.9507584601640702, + "num_tokens": 3566137.0, + "step": 2500 + }, + { + "entropy": 0.19700154662132263, + "epoch": 6.555984555984556, + "grad_norm": 0.467965304851532, + "learning_rate": 8.680674293313417e-05, + "loss": 0.14303470611572267, + "mean_token_accuracy": 0.9515972435474396, + "num_tokens": 3639573.0, + "step": 2550 + }, + { + "entropy": 0.20180423602461814, + "epoch": 6.684684684684685, + "grad_norm": 0.36836138367652893, + "learning_rate": 8.118498385878736e-05, + "loss": 0.14280882835388184, + "mean_token_accuracy": 0.9515993863344192, + "num_tokens": 3710433.0, + "step": 2600 + }, + { + "entropy": 0.20024395987391472, + "epoch": 6.813384813384813, + "grad_norm": 0.38375866413116455, + "learning_rate": 7.567249768489171e-05, + "loss": 0.1427844524383545, + "mean_token_accuracy": 0.9524166631698608, + "num_tokens": 3781550.0, + "step": 2650 + }, + { + "entropy": 0.19561587080359458, + "epoch": 6.942084942084942, + "grad_norm": 0.41185441613197327, + "learning_rate": 7.028037948510187e-05, + "loss": 0.13993803024291993, + "mean_token_accuracy": 0.9522478264570237, + "num_tokens": 3854952.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.19501976062034823, + "eval_loss": 1.0653952360153198, + "eval_mean_token_accuracy": 0.8205490803595671, + "eval_num_tokens": 3888451.0, + "eval_runtime": 161.8533, + "eval_samples_per_second": 9.546, + "eval_steps_per_second": 1.199, + "step": 2723 + }, + { + "entropy": 0.17789882526855277, + "epoch": 7.06949806949807, + "grad_norm": 0.41413992643356323, + "learning_rate": 6.50194820664261e-05, + "loss": 0.12078390121459961, + "mean_token_accuracy": 0.9589925727458916, + "num_tokens": 3928354.0, + "step": 2750 + }, + { + "entropy": 0.16781829454004765, + "epoch": 7.198198198198198, + "grad_norm": 0.25806066393852234, + "learning_rate": 5.990039412559906e-05, + "loss": 0.10963023185729981, + "mean_token_accuracy": 0.9617267113924026, + "num_tokens": 4000113.0, + "step": 2800 + }, + { + "entropy": 0.1649068508297205, + "epoch": 7.326898326898327, + "grad_norm": 0.27411890029907227, + "learning_rate": 5.493341893703393e-05, + "loss": 0.11152458190917969, + "mean_token_accuracy": 0.9620639663934708, + "num_tokens": 4071032.0, + "step": 2850 + }, + { + "entropy": 0.161333369910717, + "epoch": 7.455598455598455, + "grad_norm": 0.24944494664669037, + "learning_rate": 5.0128553615248396e-05, + "loss": 0.1094522476196289, + "mean_token_accuracy": 0.962428919672966, + "num_tokens": 4143616.0, + "step": 2900 + }, + { + "entropy": 0.15613057143986225, + "epoch": 7.584298584298584, + "grad_norm": 0.1455036848783493, + "learning_rate": 4.549546899350423e-05, + "loss": 0.11092090606689453, + "mean_token_accuracy": 0.9620462411642074, + "num_tokens": 4215664.0, + "step": 2950 + }, + { + "entropy": 0.1631234459578991, + "epoch": 7.712998712998713, + "grad_norm": 0.2129560261964798, + "learning_rate": 4.104349015915862e-05, + "loss": 0.1141857624053955, + "mean_token_accuracy": 0.9613765001296997, + "num_tokens": 4286387.0, + "step": 3000 + }, + { + "entropy": 0.1680422095954418, + "epoch": 7.841698841698841, + "grad_norm": 0.24886097013950348, + "learning_rate": 3.678157768490372e-05, + "loss": 0.11513191223144531, + "mean_token_accuracy": 0.9615794748067856, + "num_tokens": 4355875.0, + "step": 3050 + }, + { + "entropy": 0.16354035697877406, + "epoch": 7.97039897039897, + "grad_norm": 0.27600204944610596, + "learning_rate": 3.27183095936714e-05, + "loss": 0.1118631362915039, + "mean_token_accuracy": 0.9623224419355393, + "num_tokens": 4427088.0, + "step": 3100 + }, + { + "epoch": 8.0, + "eval_entropy": 0.16586771200305409, + "eval_loss": 1.204746961593628, + "eval_mean_token_accuracy": 0.8229975042883882, + "eval_num_tokens": 4443944.0, + "eval_runtime": 162.0251, + "eval_samples_per_second": 9.536, + "eval_steps_per_second": 1.197, + "step": 3112 + }, + { + "entropy": 0.1515902608934075, + "epoch": 8.097812097812097, + "grad_norm": 0.14122211933135986, + "learning_rate": 2.88618640935022e-05, + "loss": 0.09900871276855469, + "mean_token_accuracy": 0.9665110737386376, + "num_tokens": 4497867.0, + "step": 3150 + }, + { + "entropy": 0.14593622356653213, + "epoch": 8.226512226512227, + "grad_norm": 0.20527532696723938, + "learning_rate": 2.5220003117128462e-05, + "loss": 0.09842084884643555, + "mean_token_accuracy": 0.9655911487340927, + "num_tokens": 4568534.0, + "step": 3200 + }, + { + "entropy": 0.1482392605394125, + "epoch": 8.355212355212355, + "grad_norm": 0.13207173347473145, + "learning_rate": 2.1800056699401584e-05, + "loss": 0.09551989555358886, + "mean_token_accuracy": 0.9650829958915711, + "num_tokens": 4642530.0, + "step": 3250 + }, + { + "entropy": 0.15284131653606892, + "epoch": 8.483912483912484, + "grad_norm": 0.1777282953262329, + "learning_rate": 1.860890822400777e-05, + "loss": 0.10169261932373047, + "mean_token_accuracy": 0.9635212075710297, + "num_tokens": 4711573.0, + "step": 3300 + }, + { + "entropy": 0.15095721945166587, + "epoch": 8.612612612612612, + "grad_norm": 0.14988408982753754, + "learning_rate": 1.5652980569165692e-05, + "loss": 0.10045011520385742, + "mean_token_accuracy": 0.96439110994339, + "num_tokens": 4782666.0, + "step": 3350 + }, + { + "entropy": 0.15411154814064504, + "epoch": 8.741312741312742, + "grad_norm": 0.14055995643138885, + "learning_rate": 1.2938223180191691e-05, + "loss": 0.1034860897064209, + "mean_token_accuracy": 0.963447842001915, + "num_tokens": 4852180.0, + "step": 3400 + }, + { + "entropy": 0.14739766091108322, + "epoch": 8.87001287001287, + "grad_norm": 0.15041407942771912, + "learning_rate": 1.0470100094950792e-05, + "loss": 0.09690508842468262, + "mean_token_accuracy": 0.96561603307724, + "num_tokens": 4926402.0, + "step": 3450 + }, + { + "entropy": 0.1498453303426504, + "epoch": 8.998712998712998, + "grad_norm": 0.1293368935585022, + "learning_rate": 8.253578946296125e-06, + "loss": 0.09874271392822266, + "mean_token_accuracy": 0.9647125631570816, + "num_tokens": 4998841.0, + "step": 3500 + }, + { + "epoch": 9.0, + "eval_entropy": 0.151532097729211, + "eval_loss": 1.302620768547058, + "eval_mean_token_accuracy": 0.8230925079473516, + "eval_num_tokens": 4999437.0, + "eval_runtime": 161.5816, + "eval_samples_per_second": 9.562, + "eval_steps_per_second": 1.201, + "step": 3501 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.377780321673011e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cd0d97a8c611bcad7d4624b6232e1767e61fe6d3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.00237968804112545, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a0fea6bb7cc8544dfddffbbc545aca8e5f4dcbd5 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/trainer_state.json @@ -0,0 +1,115 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 389, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6110110306739807, + "epoch": 0.1287001287001287, + "grad_norm": 0.8611119389533997, + "learning_rate": 3.4130257962133866e-05, + "loss": 1.53956298828125, + "mean_token_accuracy": 0.6652342769503593, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.8662820833921433, + "epoch": 0.2574002574002574, + "grad_norm": 0.7405035495758057, + "learning_rate": 6.895705180104598e-05, + "loss": 0.7929539489746094, + "mean_token_accuracy": 0.7786347842216492, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7912819278240204, + "epoch": 0.3861003861003861, + "grad_norm": 0.6105485558509827, + "learning_rate": 0.00010378384563995809, + "loss": 0.7236511993408203, + "mean_token_accuracy": 0.7925467795133591, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7777709531784057, + "epoch": 0.5148005148005148, + "grad_norm": 0.5781793594360352, + "learning_rate": 0.0001386106394788702, + "loss": 0.7024919891357422, + "mean_token_accuracy": 0.7969876372814179, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7632609683275223, + "epoch": 0.6435006435006435, + "grad_norm": 0.6553444862365723, + "learning_rate": 0.00017343743331778232, + "loss": 0.7000718688964844, + "mean_token_accuracy": 0.7994742071628571, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.754165632724762, + "epoch": 0.7722007722007722, + "grad_norm": 0.634222149848938, + "learning_rate": 0.00020826422715669444, + "loss": 0.6909049987792969, + "mean_token_accuracy": 0.8016809666156769, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.7530711203813553, + "epoch": 0.9009009009009009, + "grad_norm": 0.7020156383514404, + "learning_rate": 0.00024309102099560653, + "loss": 0.69554931640625, + "mean_token_accuracy": 0.8002807641029358, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.5728044164242204, + "eval_loss": 0.6785851716995239, + "eval_mean_token_accuracy": 0.798433246993527, + "eval_num_tokens": 555493.0, + "eval_runtime": 162.4129, + "eval_samples_per_second": 9.513, + "eval_steps_per_second": 1.194, + "step": 389 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.306612280369152e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cd0d97a8c611bcad7d4624b6232e1767e61fe6d3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.00237968804112545, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ad2c3d366bb2a6896132cf4e21f72c1c718d2a42 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.09831666542701797, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e721fce6bf790a8cb4f7a6886d828fcad34b891b --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/trainer_state.json @@ -0,0 +1,307 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1224, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6558652856945992, + "epoch": 0.12277470841006753, + "grad_norm": 0.8294193148612976, + "learning_rate": 2.604481765338724e-05, + "loss": 1.5730807495117187, + "mean_token_accuracy": 0.6558220008015633, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.8554385647177696, + "epoch": 0.24554941682013506, + "grad_norm": 1.029894232749939, + "learning_rate": 5.2621162197659935e-05, + "loss": 0.7961322021484375, + "mean_token_accuracy": 0.7755533090233803, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7380270153284073, + "epoch": 0.3683241252302026, + "grad_norm": 0.6096898913383484, + "learning_rate": 7.919750674193263e-05, + "loss": 0.6843100738525391, + "mean_token_accuracy": 0.7998733684420586, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6932123881578446, + "epoch": 0.4910988336402701, + "grad_norm": 0.5934288501739502, + "learning_rate": 0.00010577385128620532, + "loss": 0.6458084106445312, + "mean_token_accuracy": 0.8083393195271492, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6840551143884659, + "epoch": 0.6138735420503376, + "grad_norm": 0.4665544331073761, + "learning_rate": 0.00013235019583047802, + "loss": 0.6333241653442383, + "mean_token_accuracy": 0.8122684139013291, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6649293206632138, + "epoch": 0.7366482504604052, + "grad_norm": 0.47893014550209045, + "learning_rate": 0.00015892654037475069, + "loss": 0.6107040786743164, + "mean_token_accuracy": 0.8165940269827843, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6492742404341698, + "epoch": 0.8594229588704727, + "grad_norm": 0.6059293746948242, + "learning_rate": 0.0001855028849190234, + "loss": 0.597186050415039, + "mean_token_accuracy": 0.8217281407117843, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.642182088047266, + "epoch": 0.9821976672805403, + "grad_norm": 0.3793066143989563, + "learning_rate": 0.0002120792294632961, + "loss": 0.5949800491333008, + "mean_token_accuracy": 0.8229166463017463, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6541631272860936, + "eval_loss": 0.5903413891792297, + "eval_mean_token_accuracy": 0.8252546640804835, + "eval_num_tokens": 1101768.0, + "eval_runtime": 108.9056, + "eval_samples_per_second": 12.818, + "eval_steps_per_second": 1.607, + "step": 408 + }, + { + "entropy": 0.6077755300829253, + "epoch": 1.1031307550644567, + "grad_norm": 0.5028847455978394, + "learning_rate": 0.00021679626884558217, + "loss": 0.5617346954345703, + "mean_token_accuracy": 0.8301527442665875, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5860125370323658, + "epoch": 1.2259054634745243, + "grad_norm": 0.38494572043418884, + "learning_rate": 0.00021653451093163906, + "loss": 0.5437137985229492, + "mean_token_accuracy": 0.8346565261483192, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5945393888652325, + "epoch": 1.3486801718845918, + "grad_norm": 0.5123576521873474, + "learning_rate": 0.00021607496224450087, + "loss": 0.54191650390625, + "mean_token_accuracy": 0.8335164493322372, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.6083934807777405, + "epoch": 1.4714548802946594, + "grad_norm": 0.46661439538002014, + "learning_rate": 0.000215418463597734, + "loss": 0.5550478744506836, + "mean_token_accuracy": 0.8311341696977615, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5862340961396694, + "epoch": 1.5942295887047269, + "grad_norm": 0.49572765827178955, + "learning_rate": 0.00021456621615453177, + "loss": 0.5297146606445312, + "mean_token_accuracy": 0.8360210624337197, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5832360745966434, + "epoch": 1.7170042971147943, + "grad_norm": 0.35520103573799133, + "learning_rate": 0.0002135197792300053, + "loss": 0.5287040328979492, + "mean_token_accuracy": 0.8368027776479721, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5792283065617084, + "epoch": 1.839779005524862, + "grad_norm": 0.3548352122306824, + "learning_rate": 0.00021228106743818178, + "loss": 0.5251237487792969, + "mean_token_accuracy": 0.8396679371595382, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5694268302619457, + "epoch": 1.9625537139349294, + "grad_norm": 0.3476282060146332, + "learning_rate": 0.00021085234718892933, + "loss": 0.5189918899536132, + "mean_token_accuracy": 0.8398650133609772, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5552395452771868, + "eval_loss": 0.5418588519096375, + "eval_mean_token_accuracy": 0.8379638079234532, + "eval_num_tokens": 2203536.0, + "eval_runtime": 108.6649, + "eval_samples_per_second": 12.847, + "eval_steps_per_second": 1.61, + "step": 816 + }, + { + "entropy": 0.5162206395023365, + "epoch": 2.0834868017188457, + "grad_norm": 0.3772931396961212, + "learning_rate": 0.0002092362325412188, + "loss": 0.4619992446899414, + "mean_token_accuracy": 0.8528054483650904, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.5109021583199501, + "epoch": 2.2062615101289134, + "grad_norm": 0.4661090672016144, + "learning_rate": 0.000207435680420309, + "loss": 0.4571444702148437, + "mean_token_accuracy": 0.8552933797240257, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.5012516237795352, + "epoch": 2.329036218538981, + "grad_norm": 0.5193169713020325, + "learning_rate": 0.0002054539852076065, + "loss": 0.45432735443115235, + "mean_token_accuracy": 0.8564087572693825, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.5173932652175427, + "epoch": 2.4518109269490487, + "grad_norm": 0.4666334390640259, + "learning_rate": 0.00020329477271309812, + "loss": 0.4616986083984375, + "mean_token_accuracy": 0.8532203987240792, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.5169526914507151, + "epoch": 2.574585635359116, + "grad_norm": 0.47921115159988403, + "learning_rate": 0.0002009619935413857, + "loss": 0.45737281799316404, + "mean_token_accuracy": 0.8535552659630775, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.50618562489748, + "epoch": 2.6973603437691835, + "grad_norm": 0.3407684862613678, + "learning_rate": 0.00019845991586345935, + "loss": 0.45972068786621095, + "mean_token_accuracy": 0.8532172521948814, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.5158357314765454, + "epoch": 2.820135052179251, + "grad_norm": 0.47901391983032227, + "learning_rate": 0.00019579311760743563, + "loss": 0.46119583129882813, + "mean_token_accuracy": 0.8536588314175606, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.5069115920364857, + "epoch": 2.942909760589319, + "grad_norm": 0.4287651479244232, + "learning_rate": 0.00019296647808254838, + "loss": 0.45447597503662107, + "mean_token_accuracy": 0.856622197329998, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5097703012398311, + "eval_loss": 0.5291272401809692, + "eval_mean_token_accuracy": 0.8438761404582432, + "eval_num_tokens": 3305304.0, + "eval_runtime": 108.6946, + "eval_samples_per_second": 12.843, + "eval_steps_per_second": 1.61, + "step": 1224 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.376308378051789e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ad2c3d366bb2a6896132cf4e21f72c1c718d2a42 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.09831666542701797, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6cecd3d9a1583765d4e73fb97001c91c2a371c7e --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/trainer_state.json @@ -0,0 +1,398 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1632, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6558652856945992, + "epoch": 0.12277470841006753, + "grad_norm": 0.8294193148612976, + "learning_rate": 2.604481765338724e-05, + "loss": 1.5730807495117187, + "mean_token_accuracy": 0.6558220008015633, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.8554385647177696, + "epoch": 0.24554941682013506, + "grad_norm": 1.029894232749939, + "learning_rate": 5.2621162197659935e-05, + "loss": 0.7961322021484375, + "mean_token_accuracy": 0.7755533090233803, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7380270153284073, + "epoch": 0.3683241252302026, + "grad_norm": 0.6096898913383484, + "learning_rate": 7.919750674193263e-05, + "loss": 0.6843100738525391, + "mean_token_accuracy": 0.7998733684420586, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6932123881578446, + "epoch": 0.4910988336402701, + "grad_norm": 0.5934288501739502, + "learning_rate": 0.00010577385128620532, + "loss": 0.6458084106445312, + "mean_token_accuracy": 0.8083393195271492, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6840551143884659, + "epoch": 0.6138735420503376, + "grad_norm": 0.4665544331073761, + "learning_rate": 0.00013235019583047802, + "loss": 0.6333241653442383, + "mean_token_accuracy": 0.8122684139013291, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6649293206632138, + "epoch": 0.7366482504604052, + "grad_norm": 0.47893014550209045, + "learning_rate": 0.00015892654037475069, + "loss": 0.6107040786743164, + "mean_token_accuracy": 0.8165940269827843, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6492742404341698, + "epoch": 0.8594229588704727, + "grad_norm": 0.6059293746948242, + "learning_rate": 0.0001855028849190234, + "loss": 0.597186050415039, + "mean_token_accuracy": 0.8217281407117843, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.642182088047266, + "epoch": 0.9821976672805403, + "grad_norm": 0.3793066143989563, + "learning_rate": 0.0002120792294632961, + "loss": 0.5949800491333008, + "mean_token_accuracy": 0.8229166463017463, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6541631272860936, + "eval_loss": 0.5903413891792297, + "eval_mean_token_accuracy": 0.8252546640804835, + "eval_num_tokens": 1101768.0, + "eval_runtime": 108.9056, + "eval_samples_per_second": 12.818, + "eval_steps_per_second": 1.607, + "step": 408 + }, + { + "entropy": 0.6077755300829253, + "epoch": 1.1031307550644567, + "grad_norm": 0.5028847455978394, + "learning_rate": 0.00021679626884558217, + "loss": 0.5617346954345703, + "mean_token_accuracy": 0.8301527442665875, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5860125370323658, + "epoch": 1.2259054634745243, + "grad_norm": 0.38494572043418884, + "learning_rate": 0.00021653451093163906, + "loss": 0.5437137985229492, + "mean_token_accuracy": 0.8346565261483192, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5945393888652325, + "epoch": 1.3486801718845918, + "grad_norm": 0.5123576521873474, + "learning_rate": 0.00021607496224450087, + "loss": 0.54191650390625, + "mean_token_accuracy": 0.8335164493322372, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.6083934807777405, + "epoch": 1.4714548802946594, + "grad_norm": 0.46661439538002014, + "learning_rate": 0.000215418463597734, + "loss": 0.5550478744506836, + "mean_token_accuracy": 0.8311341696977615, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5862340961396694, + "epoch": 1.5942295887047269, + "grad_norm": 0.49572765827178955, + "learning_rate": 0.00021456621615453177, + "loss": 0.5297146606445312, + "mean_token_accuracy": 0.8360210624337197, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5832360745966434, + "epoch": 1.7170042971147943, + "grad_norm": 0.35520103573799133, + "learning_rate": 0.0002135197792300053, + "loss": 0.5287040328979492, + "mean_token_accuracy": 0.8368027776479721, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5792283065617084, + "epoch": 1.839779005524862, + "grad_norm": 0.3548352122306824, + "learning_rate": 0.00021228106743818178, + "loss": 0.5251237487792969, + "mean_token_accuracy": 0.8396679371595382, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5694268302619457, + "epoch": 1.9625537139349294, + "grad_norm": 0.3476282060146332, + "learning_rate": 0.00021085234718892933, + "loss": 0.5189918899536132, + "mean_token_accuracy": 0.8398650133609772, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5552395452771868, + "eval_loss": 0.5418588519096375, + "eval_mean_token_accuracy": 0.8379638079234532, + "eval_num_tokens": 2203536.0, + "eval_runtime": 108.6649, + "eval_samples_per_second": 12.847, + "eval_steps_per_second": 1.61, + "step": 816 + }, + { + "entropy": 0.5162206395023365, + "epoch": 2.0834868017188457, + "grad_norm": 0.3772931396961212, + "learning_rate": 0.0002092362325412188, + "loss": 0.4619992446899414, + "mean_token_accuracy": 0.8528054483650904, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.5109021583199501, + "epoch": 2.2062615101289134, + "grad_norm": 0.4661090672016144, + "learning_rate": 0.000207435680420309, + "loss": 0.4571444702148437, + "mean_token_accuracy": 0.8552933797240257, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.5012516237795352, + "epoch": 2.329036218538981, + "grad_norm": 0.5193169713020325, + "learning_rate": 0.0002054539852076065, + "loss": 0.45432735443115235, + "mean_token_accuracy": 0.8564087572693825, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.5173932652175427, + "epoch": 2.4518109269490487, + "grad_norm": 0.4666334390640259, + "learning_rate": 0.00020329477271309812, + "loss": 0.4616986083984375, + "mean_token_accuracy": 0.8532203987240792, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.5169526914507151, + "epoch": 2.574585635359116, + "grad_norm": 0.47921115159988403, + "learning_rate": 0.0002009619935413857, + "loss": 0.45737281799316404, + "mean_token_accuracy": 0.8535552659630775, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.50618562489748, + "epoch": 2.6973603437691835, + "grad_norm": 0.3407684862613678, + "learning_rate": 0.00019845991586345935, + "loss": 0.45972068786621095, + "mean_token_accuracy": 0.8532172521948814, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.5158357314765454, + "epoch": 2.820135052179251, + "grad_norm": 0.47901391983032227, + "learning_rate": 0.00019579311760743563, + "loss": 0.46119583129882813, + "mean_token_accuracy": 0.8536588314175606, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.5069115920364857, + "epoch": 2.942909760589319, + "grad_norm": 0.4287651479244232, + "learning_rate": 0.00019296647808254838, + "loss": 0.45447597503662107, + "mean_token_accuracy": 0.856622197329998, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5097703012398311, + "eval_loss": 0.5291272401809692, + "eval_mean_token_accuracy": 0.8438761404582432, + "eval_num_tokens": 3305304.0, + "eval_runtime": 108.6946, + "eval_samples_per_second": 12.843, + "eval_steps_per_second": 1.61, + "step": 1224 + }, + { + "entropy": 0.4642504287250151, + "epoch": 3.063842848373235, + "grad_norm": 0.37649285793304443, + "learning_rate": 0.00018998516905171711, + "loss": 0.4056727600097656, + "mean_token_accuracy": 0.8674883682110588, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.42494223803281783, + "epoch": 3.1866175567833026, + "grad_norm": 0.38774123787879944, + "learning_rate": 0.00018685464526902752, + "loss": 0.36686546325683594, + "mean_token_accuracy": 0.8779315340518952, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.4314767476916313, + "epoch": 3.3093922651933703, + "grad_norm": 0.4668020009994507, + "learning_rate": 0.0001835806344994371, + "loss": 0.37623233795166017, + "mean_token_accuracy": 0.8753074812889099, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.4394826665520668, + "epoch": 3.4321669736034375, + "grad_norm": 0.5372388362884521, + "learning_rate": 0.00018016912703896598, + "loss": 0.3783976364135742, + "mean_token_accuracy": 0.8756387433409691, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.44438183814287185, + "epoch": 3.554941682013505, + "grad_norm": 0.40710341930389404, + "learning_rate": 0.00017662636475454779, + "loss": 0.3865407180786133, + "mean_token_accuracy": 0.8736134466528892, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.43224798239767553, + "epoch": 3.677716390423573, + "grad_norm": 0.48399633169174194, + "learning_rate": 0.0001729588296635937, + "loss": 0.3756423568725586, + "mean_token_accuracy": 0.8760525667667389, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.4290706454217434, + "epoch": 3.80049109883364, + "grad_norm": 0.46181294322013855, + "learning_rate": 0.00016917323207416466, + "loss": 0.37630157470703124, + "mean_token_accuracy": 0.8745916694402694, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.4355049130320549, + "epoch": 3.9232658072437077, + "grad_norm": 0.41401147842407227, + "learning_rate": 0.00016527649830745195, + "loss": 0.38449142456054686, + "mean_token_accuracy": 0.8750512120127678, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.458916278566633, + "eval_loss": 0.533890962600708, + "eval_mean_token_accuracy": 0.8472024192128863, + "eval_num_tokens": 4407072.0, + "eval_runtime": 108.7554, + "eval_samples_per_second": 12.836, + "eval_steps_per_second": 1.609, + "step": 1632 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.169528253786931e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ad2c3d366bb2a6896132cf4e21f72c1c718d2a42 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.09831666542701797, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d3dcff2b1f51e2366b99b1782b1f6695fb4c1fe7 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/trainer_state.json @@ -0,0 +1,489 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2040, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6558652856945992, + "epoch": 0.12277470841006753, + "grad_norm": 0.8294193148612976, + "learning_rate": 2.604481765338724e-05, + "loss": 1.5730807495117187, + "mean_token_accuracy": 0.6558220008015633, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.8554385647177696, + "epoch": 0.24554941682013506, + "grad_norm": 1.029894232749939, + "learning_rate": 5.2621162197659935e-05, + "loss": 0.7961322021484375, + "mean_token_accuracy": 0.7755533090233803, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7380270153284073, + "epoch": 0.3683241252302026, + "grad_norm": 0.6096898913383484, + "learning_rate": 7.919750674193263e-05, + "loss": 0.6843100738525391, + "mean_token_accuracy": 0.7998733684420586, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6932123881578446, + "epoch": 0.4910988336402701, + "grad_norm": 0.5934288501739502, + "learning_rate": 0.00010577385128620532, + "loss": 0.6458084106445312, + "mean_token_accuracy": 0.8083393195271492, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6840551143884659, + "epoch": 0.6138735420503376, + "grad_norm": 0.4665544331073761, + "learning_rate": 0.00013235019583047802, + "loss": 0.6333241653442383, + "mean_token_accuracy": 0.8122684139013291, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6649293206632138, + "epoch": 0.7366482504604052, + "grad_norm": 0.47893014550209045, + "learning_rate": 0.00015892654037475069, + "loss": 0.6107040786743164, + "mean_token_accuracy": 0.8165940269827843, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6492742404341698, + "epoch": 0.8594229588704727, + "grad_norm": 0.6059293746948242, + "learning_rate": 0.0001855028849190234, + "loss": 0.597186050415039, + "mean_token_accuracy": 0.8217281407117843, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.642182088047266, + "epoch": 0.9821976672805403, + "grad_norm": 0.3793066143989563, + "learning_rate": 0.0002120792294632961, + "loss": 0.5949800491333008, + "mean_token_accuracy": 0.8229166463017463, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6541631272860936, + "eval_loss": 0.5903413891792297, + "eval_mean_token_accuracy": 0.8252546640804835, + "eval_num_tokens": 1101768.0, + "eval_runtime": 108.9056, + "eval_samples_per_second": 12.818, + "eval_steps_per_second": 1.607, + "step": 408 + }, + { + "entropy": 0.6077755300829253, + "epoch": 1.1031307550644567, + "grad_norm": 0.5028847455978394, + "learning_rate": 0.00021679626884558217, + "loss": 0.5617346954345703, + "mean_token_accuracy": 0.8301527442665875, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5860125370323658, + "epoch": 1.2259054634745243, + "grad_norm": 0.38494572043418884, + "learning_rate": 0.00021653451093163906, + "loss": 0.5437137985229492, + "mean_token_accuracy": 0.8346565261483192, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5945393888652325, + "epoch": 1.3486801718845918, + "grad_norm": 0.5123576521873474, + "learning_rate": 0.00021607496224450087, + "loss": 0.54191650390625, + "mean_token_accuracy": 0.8335164493322372, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.6083934807777405, + "epoch": 1.4714548802946594, + "grad_norm": 0.46661439538002014, + "learning_rate": 0.000215418463597734, + "loss": 0.5550478744506836, + "mean_token_accuracy": 0.8311341696977615, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5862340961396694, + "epoch": 1.5942295887047269, + "grad_norm": 0.49572765827178955, + "learning_rate": 0.00021456621615453177, + "loss": 0.5297146606445312, + "mean_token_accuracy": 0.8360210624337197, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5832360745966434, + "epoch": 1.7170042971147943, + "grad_norm": 0.35520103573799133, + "learning_rate": 0.0002135197792300053, + "loss": 0.5287040328979492, + "mean_token_accuracy": 0.8368027776479721, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5792283065617084, + "epoch": 1.839779005524862, + "grad_norm": 0.3548352122306824, + "learning_rate": 0.00021228106743818178, + "loss": 0.5251237487792969, + "mean_token_accuracy": 0.8396679371595382, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5694268302619457, + "epoch": 1.9625537139349294, + "grad_norm": 0.3476282060146332, + "learning_rate": 0.00021085234718892933, + "loss": 0.5189918899536132, + "mean_token_accuracy": 0.8398650133609772, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5552395452771868, + "eval_loss": 0.5418588519096375, + "eval_mean_token_accuracy": 0.8379638079234532, + "eval_num_tokens": 2203536.0, + "eval_runtime": 108.6649, + "eval_samples_per_second": 12.847, + "eval_steps_per_second": 1.61, + "step": 816 + }, + { + "entropy": 0.5162206395023365, + "epoch": 2.0834868017188457, + "grad_norm": 0.3772931396961212, + "learning_rate": 0.0002092362325412188, + "loss": 0.4619992446899414, + "mean_token_accuracy": 0.8528054483650904, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.5109021583199501, + "epoch": 2.2062615101289134, + "grad_norm": 0.4661090672016144, + "learning_rate": 0.000207435680420309, + "loss": 0.4571444702148437, + "mean_token_accuracy": 0.8552933797240257, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.5012516237795352, + "epoch": 2.329036218538981, + "grad_norm": 0.5193169713020325, + "learning_rate": 0.0002054539852076065, + "loss": 0.45432735443115235, + "mean_token_accuracy": 0.8564087572693825, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.5173932652175427, + "epoch": 2.4518109269490487, + "grad_norm": 0.4666334390640259, + "learning_rate": 0.00020329477271309812, + "loss": 0.4616986083984375, + "mean_token_accuracy": 0.8532203987240792, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.5169526914507151, + "epoch": 2.574585635359116, + "grad_norm": 0.47921115159988403, + "learning_rate": 0.0002009619935413857, + "loss": 0.45737281799316404, + "mean_token_accuracy": 0.8535552659630775, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.50618562489748, + "epoch": 2.6973603437691835, + "grad_norm": 0.3407684862613678, + "learning_rate": 0.00019845991586345935, + "loss": 0.45972068786621095, + "mean_token_accuracy": 0.8532172521948814, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.5158357314765454, + "epoch": 2.820135052179251, + "grad_norm": 0.47901391983032227, + "learning_rate": 0.00019579311760743563, + "loss": 0.46119583129882813, + "mean_token_accuracy": 0.8536588314175606, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.5069115920364857, + "epoch": 2.942909760589319, + "grad_norm": 0.4287651479244232, + "learning_rate": 0.00019296647808254838, + "loss": 0.45447597503662107, + "mean_token_accuracy": 0.856622197329998, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5097703012398311, + "eval_loss": 0.5291272401809692, + "eval_mean_token_accuracy": 0.8438761404582432, + "eval_num_tokens": 3305304.0, + "eval_runtime": 108.6946, + "eval_samples_per_second": 12.843, + "eval_steps_per_second": 1.61, + "step": 1224 + }, + { + "entropy": 0.4642504287250151, + "epoch": 3.063842848373235, + "grad_norm": 0.37649285793304443, + "learning_rate": 0.00018998516905171711, + "loss": 0.4056727600097656, + "mean_token_accuracy": 0.8674883682110588, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.42494223803281783, + "epoch": 3.1866175567833026, + "grad_norm": 0.38774123787879944, + "learning_rate": 0.00018685464526902752, + "loss": 0.36686546325683594, + "mean_token_accuracy": 0.8779315340518952, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.4314767476916313, + "epoch": 3.3093922651933703, + "grad_norm": 0.4668020009994507, + "learning_rate": 0.0001835806344994371, + "loss": 0.37623233795166017, + "mean_token_accuracy": 0.8753074812889099, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.4394826665520668, + "epoch": 3.4321669736034375, + "grad_norm": 0.5372388362884521, + "learning_rate": 0.00018016912703896598, + "loss": 0.3783976364135742, + "mean_token_accuracy": 0.8756387433409691, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.44438183814287185, + "epoch": 3.554941682013505, + "grad_norm": 0.40710341930389404, + "learning_rate": 0.00017662636475454779, + "loss": 0.3865407180786133, + "mean_token_accuracy": 0.8736134466528892, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.43224798239767553, + "epoch": 3.677716390423573, + "grad_norm": 0.48399633169174194, + "learning_rate": 0.0001729588296635937, + "loss": 0.3756423568725586, + "mean_token_accuracy": 0.8760525667667389, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.4290706454217434, + "epoch": 3.80049109883364, + "grad_norm": 0.46181294322013855, + "learning_rate": 0.00016917323207416466, + "loss": 0.37630157470703124, + "mean_token_accuracy": 0.8745916694402694, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.4355049130320549, + "epoch": 3.9232658072437077, + "grad_norm": 0.41401147842407227, + "learning_rate": 0.00016527649830745195, + "loss": 0.38449142456054686, + "mean_token_accuracy": 0.8750512120127678, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.458916278566633, + "eval_loss": 0.533890962600708, + "eval_mean_token_accuracy": 0.8472024192128863, + "eval_num_tokens": 4407072.0, + "eval_runtime": 108.7554, + "eval_samples_per_second": 12.836, + "eval_steps_per_second": 1.609, + "step": 1632 + }, + { + "entropy": 0.38954928343368667, + "epoch": 4.044198895027624, + "grad_norm": 0.4088170826435089, + "learning_rate": 0.0001612757580250288, + "loss": 0.3357394790649414, + "mean_token_accuracy": 0.8872706052010435, + "num_tokens": 4455549.0, + "step": 1650 + }, + { + "entropy": 0.3478683318942785, + "epoch": 4.166973603437691, + "grad_norm": 0.65646892786026, + "learning_rate": 0.00015717833118406024, + "loss": 0.28736536026000975, + "mean_token_accuracy": 0.90216373950243, + "num_tokens": 4584218.0, + "step": 1700 + }, + { + "entropy": 0.3364737436920404, + "epoch": 4.2897483118477595, + "grad_norm": 0.5015673041343689, + "learning_rate": 0.0001529917146443387, + "loss": 0.28391304016113283, + "mean_token_accuracy": 0.9025878021121025, + "num_tokens": 4724425.0, + "step": 1750 + }, + { + "entropy": 0.34254246324300763, + "epoch": 4.412523020257827, + "grad_norm": 0.4893321096897125, + "learning_rate": 0.00014872356845164908, + "loss": 0.28648223876953127, + "mean_token_accuracy": 0.9010315355658531, + "num_tokens": 4861552.0, + "step": 1800 + }, + { + "entropy": 0.35019800525158645, + "epoch": 4.535297728667894, + "grad_norm": 0.5732161998748779, + "learning_rate": 0.0001443817018225613, + "loss": 0.2940273475646973, + "mean_token_accuracy": 0.8994512927532196, + "num_tokens": 4993610.0, + "step": 1850 + }, + { + "entropy": 0.3470254339277744, + "epoch": 4.658072437077962, + "grad_norm": 0.5012240409851074, + "learning_rate": 0.0001399740588562913, + "loss": 0.2919456481933594, + "mean_token_accuracy": 0.8995206660032272, + "num_tokens": 5126291.0, + "step": 1900 + }, + { + "entropy": 0.3445418912172318, + "epoch": 4.780847145488029, + "grad_norm": 0.49685943126678467, + "learning_rate": 0.00013550870399977507, + "loss": 0.2901294136047363, + "mean_token_accuracy": 0.9009958255290985, + "num_tokens": 5266076.0, + "step": 1950 + }, + { + "entropy": 0.3482987853884697, + "epoch": 4.903621853898097, + "grad_norm": 0.46585920453071594, + "learning_rate": 0.00013099380729254763, + "loss": 0.29066293716430663, + "mean_token_accuracy": 0.8992295026779175, + "num_tokens": 5403761.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3964379857267652, + "eval_loss": 0.5762408971786499, + "eval_mean_token_accuracy": 0.8455211687088012, + "eval_num_tokens": 5508840.0, + "eval_runtime": 108.6475, + "eval_samples_per_second": 12.849, + "eval_steps_per_second": 1.611, + "step": 2040 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.959427377074586e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ad2c3d366bb2a6896132cf4e21f72c1c718d2a42 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.09831666542701797, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..df250e4493d925b778f258729628118aa444714a --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/trainer_state.json @@ -0,0 +1,580 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2448, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6558652856945992, + "epoch": 0.12277470841006753, + "grad_norm": 0.8294193148612976, + "learning_rate": 2.604481765338724e-05, + "loss": 1.5730807495117187, + "mean_token_accuracy": 0.6558220008015633, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.8554385647177696, + "epoch": 0.24554941682013506, + "grad_norm": 1.029894232749939, + "learning_rate": 5.2621162197659935e-05, + "loss": 0.7961322021484375, + "mean_token_accuracy": 0.7755533090233803, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7380270153284073, + "epoch": 0.3683241252302026, + "grad_norm": 0.6096898913383484, + "learning_rate": 7.919750674193263e-05, + "loss": 0.6843100738525391, + "mean_token_accuracy": 0.7998733684420586, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6932123881578446, + "epoch": 0.4910988336402701, + "grad_norm": 0.5934288501739502, + "learning_rate": 0.00010577385128620532, + "loss": 0.6458084106445312, + "mean_token_accuracy": 0.8083393195271492, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6840551143884659, + "epoch": 0.6138735420503376, + "grad_norm": 0.4665544331073761, + "learning_rate": 0.00013235019583047802, + "loss": 0.6333241653442383, + "mean_token_accuracy": 0.8122684139013291, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6649293206632138, + "epoch": 0.7366482504604052, + "grad_norm": 0.47893014550209045, + "learning_rate": 0.00015892654037475069, + "loss": 0.6107040786743164, + "mean_token_accuracy": 0.8165940269827843, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6492742404341698, + "epoch": 0.8594229588704727, + "grad_norm": 0.6059293746948242, + "learning_rate": 0.0001855028849190234, + "loss": 0.597186050415039, + "mean_token_accuracy": 0.8217281407117843, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.642182088047266, + "epoch": 0.9821976672805403, + "grad_norm": 0.3793066143989563, + "learning_rate": 0.0002120792294632961, + "loss": 0.5949800491333008, + "mean_token_accuracy": 0.8229166463017463, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6541631272860936, + "eval_loss": 0.5903413891792297, + "eval_mean_token_accuracy": 0.8252546640804835, + "eval_num_tokens": 1101768.0, + "eval_runtime": 108.9056, + "eval_samples_per_second": 12.818, + "eval_steps_per_second": 1.607, + "step": 408 + }, + { + "entropy": 0.6077755300829253, + "epoch": 1.1031307550644567, + "grad_norm": 0.5028847455978394, + "learning_rate": 0.00021679626884558217, + "loss": 0.5617346954345703, + "mean_token_accuracy": 0.8301527442665875, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5860125370323658, + "epoch": 1.2259054634745243, + "grad_norm": 0.38494572043418884, + "learning_rate": 0.00021653451093163906, + "loss": 0.5437137985229492, + "mean_token_accuracy": 0.8346565261483192, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5945393888652325, + "epoch": 1.3486801718845918, + "grad_norm": 0.5123576521873474, + "learning_rate": 0.00021607496224450087, + "loss": 0.54191650390625, + "mean_token_accuracy": 0.8335164493322372, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.6083934807777405, + "epoch": 1.4714548802946594, + "grad_norm": 0.46661439538002014, + "learning_rate": 0.000215418463597734, + "loss": 0.5550478744506836, + "mean_token_accuracy": 0.8311341696977615, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5862340961396694, + "epoch": 1.5942295887047269, + "grad_norm": 0.49572765827178955, + "learning_rate": 0.00021456621615453177, + "loss": 0.5297146606445312, + "mean_token_accuracy": 0.8360210624337197, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5832360745966434, + "epoch": 1.7170042971147943, + "grad_norm": 0.35520103573799133, + "learning_rate": 0.0002135197792300053, + "loss": 0.5287040328979492, + "mean_token_accuracy": 0.8368027776479721, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5792283065617084, + "epoch": 1.839779005524862, + "grad_norm": 0.3548352122306824, + "learning_rate": 0.00021228106743818178, + "loss": 0.5251237487792969, + "mean_token_accuracy": 0.8396679371595382, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5694268302619457, + "epoch": 1.9625537139349294, + "grad_norm": 0.3476282060146332, + "learning_rate": 0.00021085234718892933, + "loss": 0.5189918899536132, + "mean_token_accuracy": 0.8398650133609772, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5552395452771868, + "eval_loss": 0.5418588519096375, + "eval_mean_token_accuracy": 0.8379638079234532, + "eval_num_tokens": 2203536.0, + "eval_runtime": 108.6649, + "eval_samples_per_second": 12.847, + "eval_steps_per_second": 1.61, + "step": 816 + }, + { + "entropy": 0.5162206395023365, + "epoch": 2.0834868017188457, + "grad_norm": 0.3772931396961212, + "learning_rate": 0.0002092362325412188, + "loss": 0.4619992446899414, + "mean_token_accuracy": 0.8528054483650904, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.5109021583199501, + "epoch": 2.2062615101289134, + "grad_norm": 0.4661090672016144, + "learning_rate": 0.000207435680420309, + "loss": 0.4571444702148437, + "mean_token_accuracy": 0.8552933797240257, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.5012516237795352, + "epoch": 2.329036218538981, + "grad_norm": 0.5193169713020325, + "learning_rate": 0.0002054539852076065, + "loss": 0.45432735443115235, + "mean_token_accuracy": 0.8564087572693825, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.5173932652175427, + "epoch": 2.4518109269490487, + "grad_norm": 0.4666334390640259, + "learning_rate": 0.00020329477271309812, + "loss": 0.4616986083984375, + "mean_token_accuracy": 0.8532203987240792, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.5169526914507151, + "epoch": 2.574585635359116, + "grad_norm": 0.47921115159988403, + "learning_rate": 0.0002009619935413857, + "loss": 0.45737281799316404, + "mean_token_accuracy": 0.8535552659630775, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.50618562489748, + "epoch": 2.6973603437691835, + "grad_norm": 0.3407684862613678, + "learning_rate": 0.00019845991586345935, + "loss": 0.45972068786621095, + "mean_token_accuracy": 0.8532172521948814, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.5158357314765454, + "epoch": 2.820135052179251, + "grad_norm": 0.47901391983032227, + "learning_rate": 0.00019579311760743563, + "loss": 0.46119583129882813, + "mean_token_accuracy": 0.8536588314175606, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.5069115920364857, + "epoch": 2.942909760589319, + "grad_norm": 0.4287651479244232, + "learning_rate": 0.00019296647808254838, + "loss": 0.45447597503662107, + "mean_token_accuracy": 0.856622197329998, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5097703012398311, + "eval_loss": 0.5291272401809692, + "eval_mean_token_accuracy": 0.8438761404582432, + "eval_num_tokens": 3305304.0, + "eval_runtime": 108.6946, + "eval_samples_per_second": 12.843, + "eval_steps_per_second": 1.61, + "step": 1224 + }, + { + "entropy": 0.4642504287250151, + "epoch": 3.063842848373235, + "grad_norm": 0.37649285793304443, + "learning_rate": 0.00018998516905171711, + "loss": 0.4056727600097656, + "mean_token_accuracy": 0.8674883682110588, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.42494223803281783, + "epoch": 3.1866175567833026, + "grad_norm": 0.38774123787879944, + "learning_rate": 0.00018685464526902752, + "loss": 0.36686546325683594, + "mean_token_accuracy": 0.8779315340518952, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.4314767476916313, + "epoch": 3.3093922651933703, + "grad_norm": 0.4668020009994507, + "learning_rate": 0.0001835806344994371, + "loss": 0.37623233795166017, + "mean_token_accuracy": 0.8753074812889099, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.4394826665520668, + "epoch": 3.4321669736034375, + "grad_norm": 0.5372388362884521, + "learning_rate": 0.00018016912703896598, + "loss": 0.3783976364135742, + "mean_token_accuracy": 0.8756387433409691, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.44438183814287185, + "epoch": 3.554941682013505, + "grad_norm": 0.40710341930389404, + "learning_rate": 0.00017662636475454779, + "loss": 0.3865407180786133, + "mean_token_accuracy": 0.8736134466528892, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.43224798239767553, + "epoch": 3.677716390423573, + "grad_norm": 0.48399633169174194, + "learning_rate": 0.0001729588296635937, + "loss": 0.3756423568725586, + "mean_token_accuracy": 0.8760525667667389, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.4290706454217434, + "epoch": 3.80049109883364, + "grad_norm": 0.46181294322013855, + "learning_rate": 0.00016917323207416466, + "loss": 0.37630157470703124, + "mean_token_accuracy": 0.8745916694402694, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.4355049130320549, + "epoch": 3.9232658072437077, + "grad_norm": 0.41401147842407227, + "learning_rate": 0.00016527649830745195, + "loss": 0.38449142456054686, + "mean_token_accuracy": 0.8750512120127678, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.458916278566633, + "eval_loss": 0.533890962600708, + "eval_mean_token_accuracy": 0.8472024192128863, + "eval_num_tokens": 4407072.0, + "eval_runtime": 108.7554, + "eval_samples_per_second": 12.836, + "eval_steps_per_second": 1.609, + "step": 1632 + }, + { + "entropy": 0.38954928343368667, + "epoch": 4.044198895027624, + "grad_norm": 0.4088170826435089, + "learning_rate": 0.0001612757580250288, + "loss": 0.3357394790649414, + "mean_token_accuracy": 0.8872706052010435, + "num_tokens": 4455549.0, + "step": 1650 + }, + { + "entropy": 0.3478683318942785, + "epoch": 4.166973603437691, + "grad_norm": 0.65646892786026, + "learning_rate": 0.00015717833118406024, + "loss": 0.28736536026000975, + "mean_token_accuracy": 0.90216373950243, + "num_tokens": 4584218.0, + "step": 1700 + }, + { + "entropy": 0.3364737436920404, + "epoch": 4.2897483118477595, + "grad_norm": 0.5015673041343689, + "learning_rate": 0.0001529917146443387, + "loss": 0.28391304016113283, + "mean_token_accuracy": 0.9025878021121025, + "num_tokens": 4724425.0, + "step": 1750 + }, + { + "entropy": 0.34254246324300763, + "epoch": 4.412523020257827, + "grad_norm": 0.4893321096897125, + "learning_rate": 0.00014872356845164908, + "loss": 0.28648223876953127, + "mean_token_accuracy": 0.9010315355658531, + "num_tokens": 4861552.0, + "step": 1800 + }, + { + "entropy": 0.35019800525158645, + "epoch": 4.535297728667894, + "grad_norm": 0.5732161998748779, + "learning_rate": 0.0001443817018225613, + "loss": 0.2940273475646973, + "mean_token_accuracy": 0.8994512927532196, + "num_tokens": 4993610.0, + "step": 1850 + }, + { + "entropy": 0.3470254339277744, + "epoch": 4.658072437077962, + "grad_norm": 0.5012240409851074, + "learning_rate": 0.0001399740588562913, + "loss": 0.2919456481933594, + "mean_token_accuracy": 0.8995206660032272, + "num_tokens": 5126291.0, + "step": 1900 + }, + { + "entropy": 0.3445418912172318, + "epoch": 4.780847145488029, + "grad_norm": 0.49685943126678467, + "learning_rate": 0.00013550870399977507, + "loss": 0.2901294136047363, + "mean_token_accuracy": 0.9009958255290985, + "num_tokens": 5266076.0, + "step": 1950 + }, + { + "entropy": 0.3482987853884697, + "epoch": 4.903621853898097, + "grad_norm": 0.46585920453071594, + "learning_rate": 0.00013099380729254763, + "loss": 0.29066293716430663, + "mean_token_accuracy": 0.8992295026779175, + "num_tokens": 5403761.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3964379857267652, + "eval_loss": 0.5762408971786499, + "eval_mean_token_accuracy": 0.8455211687088012, + "eval_num_tokens": 5508840.0, + "eval_runtime": 108.6475, + "eval_samples_per_second": 12.849, + "eval_steps_per_second": 1.611, + "step": 2040 + }, + { + "entropy": 0.33933334491277106, + "epoch": 5.024554941682013, + "grad_norm": 0.718194305896759, + "learning_rate": 0.00012643762941842452, + "loss": 0.2779405975341797, + "mean_token_accuracy": 0.90495823754877, + "num_tokens": 5535743.0, + "step": 2050 + }, + { + "entropy": 0.25700028784573076, + "epoch": 5.147329650092081, + "grad_norm": 0.5866787433624268, + "learning_rate": 0.00012184850659133646, + "loss": 0.19206731796264648, + "mean_token_accuracy": 0.9317478793859482, + "num_tokens": 5675294.0, + "step": 2100 + }, + { + "entropy": 0.26606475934386253, + "epoch": 5.270104358502149, + "grad_norm": 0.5036929845809937, + "learning_rate": 0.00011723483530296943, + "loss": 0.20060159683227538, + "mean_token_accuracy": 0.928264719247818, + "num_tokens": 5808279.0, + "step": 2150 + }, + { + "entropy": 0.2522051434218884, + "epoch": 5.392879066912216, + "grad_norm": 0.5887715220451355, + "learning_rate": 0.00011260505696011844, + "loss": 0.19347431182861327, + "mean_token_accuracy": 0.930549119412899, + "num_tokens": 5948774.0, + "step": 2200 + }, + { + "entropy": 0.26358571454882623, + "epoch": 5.515653775322283, + "grad_norm": 0.5937269330024719, + "learning_rate": 0.00010796764243986134, + "loss": 0.20256309509277343, + "mean_token_accuracy": 0.9277606004476547, + "num_tokens": 6081211.0, + "step": 2250 + }, + { + "entropy": 0.2550286700576544, + "epoch": 5.638428483732351, + "grad_norm": 0.6088381409645081, + "learning_rate": 0.00010333107659081343, + "loss": 0.19648813247680663, + "mean_token_accuracy": 0.9293895545601845, + "num_tokens": 6213793.0, + "step": 2300 + }, + { + "entropy": 0.262065271884203, + "epoch": 5.7612031921424185, + "grad_norm": 0.5331980586051941, + "learning_rate": 9.870384270881798e-05, + "loss": 0.2019593048095703, + "mean_token_accuracy": 0.9276418074965477, + "num_tokens": 6350388.0, + "step": 2350 + }, + { + "entropy": 0.2581966924667358, + "epoch": 5.883977900552486, + "grad_norm": 0.6100393533706665, + "learning_rate": 9.409440701547846e-05, + "loss": 0.20058013916015624, + "mean_token_accuracy": 0.928467052578926, + "num_tokens": 6485960.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.31780434923512596, + "eval_loss": 0.6719555258750916, + "eval_mean_token_accuracy": 0.8434631654194423, + "eval_num_tokens": 6610608.0, + "eval_runtime": 108.6811, + "eval_samples_per_second": 12.845, + "eval_steps_per_second": 1.61, + "step": 2448 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0749230774590157e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ad2c3d366bb2a6896132cf4e21f72c1c718d2a42 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.09831666542701797, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7bdd3ab1a13ed1ecb94972aca53c71da000eb6f3 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/trainer_state.json @@ -0,0 +1,681 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2856, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6558652856945992, + "epoch": 0.12277470841006753, + "grad_norm": 0.8294193148612976, + "learning_rate": 2.604481765338724e-05, + "loss": 1.5730807495117187, + "mean_token_accuracy": 0.6558220008015633, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.8554385647177696, + "epoch": 0.24554941682013506, + "grad_norm": 1.029894232749939, + "learning_rate": 5.2621162197659935e-05, + "loss": 0.7961322021484375, + "mean_token_accuracy": 0.7755533090233803, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7380270153284073, + "epoch": 0.3683241252302026, + "grad_norm": 0.6096898913383484, + "learning_rate": 7.919750674193263e-05, + "loss": 0.6843100738525391, + "mean_token_accuracy": 0.7998733684420586, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6932123881578446, + "epoch": 0.4910988336402701, + "grad_norm": 0.5934288501739502, + "learning_rate": 0.00010577385128620532, + "loss": 0.6458084106445312, + "mean_token_accuracy": 0.8083393195271492, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6840551143884659, + "epoch": 0.6138735420503376, + "grad_norm": 0.4665544331073761, + "learning_rate": 0.00013235019583047802, + "loss": 0.6333241653442383, + "mean_token_accuracy": 0.8122684139013291, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6649293206632138, + "epoch": 0.7366482504604052, + "grad_norm": 0.47893014550209045, + "learning_rate": 0.00015892654037475069, + "loss": 0.6107040786743164, + "mean_token_accuracy": 0.8165940269827843, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6492742404341698, + "epoch": 0.8594229588704727, + "grad_norm": 0.6059293746948242, + "learning_rate": 0.0001855028849190234, + "loss": 0.597186050415039, + "mean_token_accuracy": 0.8217281407117843, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.642182088047266, + "epoch": 0.9821976672805403, + "grad_norm": 0.3793066143989563, + "learning_rate": 0.0002120792294632961, + "loss": 0.5949800491333008, + "mean_token_accuracy": 0.8229166463017463, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6541631272860936, + "eval_loss": 0.5903413891792297, + "eval_mean_token_accuracy": 0.8252546640804835, + "eval_num_tokens": 1101768.0, + "eval_runtime": 108.9056, + "eval_samples_per_second": 12.818, + "eval_steps_per_second": 1.607, + "step": 408 + }, + { + "entropy": 0.6077755300829253, + "epoch": 1.1031307550644567, + "grad_norm": 0.5028847455978394, + "learning_rate": 0.00021679626884558217, + "loss": 0.5617346954345703, + "mean_token_accuracy": 0.8301527442665875, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5860125370323658, + "epoch": 1.2259054634745243, + "grad_norm": 0.38494572043418884, + "learning_rate": 0.00021653451093163906, + "loss": 0.5437137985229492, + "mean_token_accuracy": 0.8346565261483192, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5945393888652325, + "epoch": 1.3486801718845918, + "grad_norm": 0.5123576521873474, + "learning_rate": 0.00021607496224450087, + "loss": 0.54191650390625, + "mean_token_accuracy": 0.8335164493322372, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.6083934807777405, + "epoch": 1.4714548802946594, + "grad_norm": 0.46661439538002014, + "learning_rate": 0.000215418463597734, + "loss": 0.5550478744506836, + "mean_token_accuracy": 0.8311341696977615, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5862340961396694, + "epoch": 1.5942295887047269, + "grad_norm": 0.49572765827178955, + "learning_rate": 0.00021456621615453177, + "loss": 0.5297146606445312, + "mean_token_accuracy": 0.8360210624337197, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5832360745966434, + "epoch": 1.7170042971147943, + "grad_norm": 0.35520103573799133, + "learning_rate": 0.0002135197792300053, + "loss": 0.5287040328979492, + "mean_token_accuracy": 0.8368027776479721, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5792283065617084, + "epoch": 1.839779005524862, + "grad_norm": 0.3548352122306824, + "learning_rate": 0.00021228106743818178, + "loss": 0.5251237487792969, + "mean_token_accuracy": 0.8396679371595382, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5694268302619457, + "epoch": 1.9625537139349294, + "grad_norm": 0.3476282060146332, + "learning_rate": 0.00021085234718892933, + "loss": 0.5189918899536132, + "mean_token_accuracy": 0.8398650133609772, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5552395452771868, + "eval_loss": 0.5418588519096375, + "eval_mean_token_accuracy": 0.8379638079234532, + "eval_num_tokens": 2203536.0, + "eval_runtime": 108.6649, + "eval_samples_per_second": 12.847, + "eval_steps_per_second": 1.61, + "step": 816 + }, + { + "entropy": 0.5162206395023365, + "epoch": 2.0834868017188457, + "grad_norm": 0.3772931396961212, + "learning_rate": 0.0002092362325412188, + "loss": 0.4619992446899414, + "mean_token_accuracy": 0.8528054483650904, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.5109021583199501, + "epoch": 2.2062615101289134, + "grad_norm": 0.4661090672016144, + "learning_rate": 0.000207435680420309, + "loss": 0.4571444702148437, + "mean_token_accuracy": 0.8552933797240257, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.5012516237795352, + "epoch": 2.329036218538981, + "grad_norm": 0.5193169713020325, + "learning_rate": 0.0002054539852076065, + "loss": 0.45432735443115235, + "mean_token_accuracy": 0.8564087572693825, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.5173932652175427, + "epoch": 2.4518109269490487, + "grad_norm": 0.4666334390640259, + "learning_rate": 0.00020329477271309812, + "loss": 0.4616986083984375, + "mean_token_accuracy": 0.8532203987240792, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.5169526914507151, + "epoch": 2.574585635359116, + "grad_norm": 0.47921115159988403, + "learning_rate": 0.0002009619935413857, + "loss": 0.45737281799316404, + "mean_token_accuracy": 0.8535552659630775, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.50618562489748, + "epoch": 2.6973603437691835, + "grad_norm": 0.3407684862613678, + "learning_rate": 0.00019845991586345935, + "loss": 0.45972068786621095, + "mean_token_accuracy": 0.8532172521948814, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.5158357314765454, + "epoch": 2.820135052179251, + "grad_norm": 0.47901391983032227, + "learning_rate": 0.00019579311760743563, + "loss": 0.46119583129882813, + "mean_token_accuracy": 0.8536588314175606, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.5069115920364857, + "epoch": 2.942909760589319, + "grad_norm": 0.4287651479244232, + "learning_rate": 0.00019296647808254838, + "loss": 0.45447597503662107, + "mean_token_accuracy": 0.856622197329998, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5097703012398311, + "eval_loss": 0.5291272401809692, + "eval_mean_token_accuracy": 0.8438761404582432, + "eval_num_tokens": 3305304.0, + "eval_runtime": 108.6946, + "eval_samples_per_second": 12.843, + "eval_steps_per_second": 1.61, + "step": 1224 + }, + { + "entropy": 0.4642504287250151, + "epoch": 3.063842848373235, + "grad_norm": 0.37649285793304443, + "learning_rate": 0.00018998516905171711, + "loss": 0.4056727600097656, + "mean_token_accuracy": 0.8674883682110588, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.42494223803281783, + "epoch": 3.1866175567833026, + "grad_norm": 0.38774123787879944, + "learning_rate": 0.00018685464526902752, + "loss": 0.36686546325683594, + "mean_token_accuracy": 0.8779315340518952, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.4314767476916313, + "epoch": 3.3093922651933703, + "grad_norm": 0.4668020009994507, + "learning_rate": 0.0001835806344994371, + "loss": 0.37623233795166017, + "mean_token_accuracy": 0.8753074812889099, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.4394826665520668, + "epoch": 3.4321669736034375, + "grad_norm": 0.5372388362884521, + "learning_rate": 0.00018016912703896598, + "loss": 0.3783976364135742, + "mean_token_accuracy": 0.8756387433409691, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.44438183814287185, + "epoch": 3.554941682013505, + "grad_norm": 0.40710341930389404, + "learning_rate": 0.00017662636475454779, + "loss": 0.3865407180786133, + "mean_token_accuracy": 0.8736134466528892, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.43224798239767553, + "epoch": 3.677716390423573, + "grad_norm": 0.48399633169174194, + "learning_rate": 0.0001729588296635937, + "loss": 0.3756423568725586, + "mean_token_accuracy": 0.8760525667667389, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.4290706454217434, + "epoch": 3.80049109883364, + "grad_norm": 0.46181294322013855, + "learning_rate": 0.00016917323207416466, + "loss": 0.37630157470703124, + "mean_token_accuracy": 0.8745916694402694, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.4355049130320549, + "epoch": 3.9232658072437077, + "grad_norm": 0.41401147842407227, + "learning_rate": 0.00016527649830745195, + "loss": 0.38449142456054686, + "mean_token_accuracy": 0.8750512120127678, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.458916278566633, + "eval_loss": 0.533890962600708, + "eval_mean_token_accuracy": 0.8472024192128863, + "eval_num_tokens": 4407072.0, + "eval_runtime": 108.7554, + "eval_samples_per_second": 12.836, + "eval_steps_per_second": 1.609, + "step": 1632 + }, + { + "entropy": 0.38954928343368667, + "epoch": 4.044198895027624, + "grad_norm": 0.4088170826435089, + "learning_rate": 0.0001612757580250288, + "loss": 0.3357394790649414, + "mean_token_accuracy": 0.8872706052010435, + "num_tokens": 4455549.0, + "step": 1650 + }, + { + "entropy": 0.3478683318942785, + "epoch": 4.166973603437691, + "grad_norm": 0.65646892786026, + "learning_rate": 0.00015717833118406024, + "loss": 0.28736536026000975, + "mean_token_accuracy": 0.90216373950243, + "num_tokens": 4584218.0, + "step": 1700 + }, + { + "entropy": 0.3364737436920404, + "epoch": 4.2897483118477595, + "grad_norm": 0.5015673041343689, + "learning_rate": 0.0001529917146443387, + "loss": 0.28391304016113283, + "mean_token_accuracy": 0.9025878021121025, + "num_tokens": 4724425.0, + "step": 1750 + }, + { + "entropy": 0.34254246324300763, + "epoch": 4.412523020257827, + "grad_norm": 0.4893321096897125, + "learning_rate": 0.00014872356845164908, + "loss": 0.28648223876953127, + "mean_token_accuracy": 0.9010315355658531, + "num_tokens": 4861552.0, + "step": 1800 + }, + { + "entropy": 0.35019800525158645, + "epoch": 4.535297728667894, + "grad_norm": 0.5732161998748779, + "learning_rate": 0.0001443817018225613, + "loss": 0.2940273475646973, + "mean_token_accuracy": 0.8994512927532196, + "num_tokens": 4993610.0, + "step": 1850 + }, + { + "entropy": 0.3470254339277744, + "epoch": 4.658072437077962, + "grad_norm": 0.5012240409851074, + "learning_rate": 0.0001399740588562913, + "loss": 0.2919456481933594, + "mean_token_accuracy": 0.8995206660032272, + "num_tokens": 5126291.0, + "step": 1900 + }, + { + "entropy": 0.3445418912172318, + "epoch": 4.780847145488029, + "grad_norm": 0.49685943126678467, + "learning_rate": 0.00013550870399977507, + "loss": 0.2901294136047363, + "mean_token_accuracy": 0.9009958255290985, + "num_tokens": 5266076.0, + "step": 1950 + }, + { + "entropy": 0.3482987853884697, + "epoch": 4.903621853898097, + "grad_norm": 0.46585920453071594, + "learning_rate": 0.00013099380729254763, + "loss": 0.29066293716430663, + "mean_token_accuracy": 0.8992295026779175, + "num_tokens": 5403761.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3964379857267652, + "eval_loss": 0.5762408971786499, + "eval_mean_token_accuracy": 0.8455211687088012, + "eval_num_tokens": 5508840.0, + "eval_runtime": 108.6475, + "eval_samples_per_second": 12.849, + "eval_steps_per_second": 1.611, + "step": 2040 + }, + { + "entropy": 0.33933334491277106, + "epoch": 5.024554941682013, + "grad_norm": 0.718194305896759, + "learning_rate": 0.00012643762941842452, + "loss": 0.2779405975341797, + "mean_token_accuracy": 0.90495823754877, + "num_tokens": 5535743.0, + "step": 2050 + }, + { + "entropy": 0.25700028784573076, + "epoch": 5.147329650092081, + "grad_norm": 0.5866787433624268, + "learning_rate": 0.00012184850659133646, + "loss": 0.19206731796264648, + "mean_token_accuracy": 0.9317478793859482, + "num_tokens": 5675294.0, + "step": 2100 + }, + { + "entropy": 0.26606475934386253, + "epoch": 5.270104358502149, + "grad_norm": 0.5036929845809937, + "learning_rate": 0.00011723483530296943, + "loss": 0.20060159683227538, + "mean_token_accuracy": 0.928264719247818, + "num_tokens": 5808279.0, + "step": 2150 + }, + { + "entropy": 0.2522051434218884, + "epoch": 5.392879066912216, + "grad_norm": 0.5887715220451355, + "learning_rate": 0.00011260505696011844, + "loss": 0.19347431182861327, + "mean_token_accuracy": 0.930549119412899, + "num_tokens": 5948774.0, + "step": 2200 + }, + { + "entropy": 0.26358571454882623, + "epoch": 5.515653775322283, + "grad_norm": 0.5937269330024719, + "learning_rate": 0.00010796764243986134, + "loss": 0.20256309509277343, + "mean_token_accuracy": 0.9277606004476547, + "num_tokens": 6081211.0, + "step": 2250 + }, + { + "entropy": 0.2550286700576544, + "epoch": 5.638428483732351, + "grad_norm": 0.6088381409645081, + "learning_rate": 0.00010333107659081343, + "loss": 0.19648813247680663, + "mean_token_accuracy": 0.9293895545601845, + "num_tokens": 6213793.0, + "step": 2300 + }, + { + "entropy": 0.262065271884203, + "epoch": 5.7612031921424185, + "grad_norm": 0.5331980586051941, + "learning_rate": 9.870384270881798e-05, + "loss": 0.2019593048095703, + "mean_token_accuracy": 0.9276418074965477, + "num_tokens": 6350388.0, + "step": 2350 + }, + { + "entropy": 0.2581966924667358, + "epoch": 5.883977900552486, + "grad_norm": 0.6100393533706665, + "learning_rate": 9.409440701547846e-05, + "loss": 0.20058013916015624, + "mean_token_accuracy": 0.928467052578926, + "num_tokens": 6485960.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.31780434923512596, + "eval_loss": 0.6719555258750916, + "eval_mean_token_accuracy": 0.8434631654194423, + "eval_num_tokens": 6610608.0, + "eval_runtime": 108.6811, + "eval_samples_per_second": 12.845, + "eval_steps_per_second": 1.61, + "step": 2448 + }, + { + "entropy": 0.2578806725855406, + "epoch": 6.004910988336403, + "grad_norm": 0.40365952253341675, + "learning_rate": 8.951120316793048e-05, + "loss": 0.19516365051269532, + "mean_token_accuracy": 0.930728108749777, + "num_tokens": 6616117.0, + "step": 2450 + }, + { + "entropy": 0.18210563194006682, + "epoch": 6.12768569674647, + "grad_norm": 0.5063093304634094, + "learning_rate": 8.496261682819517e-05, + "loss": 0.11744135856628418, + "mean_token_accuracy": 0.9580859485268592, + "num_tokens": 6754569.0, + "step": 2500 + }, + { + "entropy": 0.18104823548346757, + "epoch": 6.250460405156538, + "grad_norm": 0.5599386096000671, + "learning_rate": 8.045697032034715e-05, + "loss": 0.11880653381347656, + "mean_token_accuracy": 0.9568264433741569, + "num_tokens": 6893447.0, + "step": 2550 + }, + { + "entropy": 0.18421781908720733, + "epoch": 6.373235113566605, + "grad_norm": 0.5448400378227234, + "learning_rate": 7.600250740356843e-05, + "loss": 0.11898751258850097, + "mean_token_accuracy": 0.9573441597819329, + "num_tokens": 7025939.0, + "step": 2600 + }, + { + "entropy": 0.18598256688565015, + "epoch": 6.496009821976672, + "grad_norm": 0.49465450644493103, + "learning_rate": 7.16073781889491e-05, + "loss": 0.11917055130004883, + "mean_token_accuracy": 0.9567968577146531, + "num_tokens": 7163978.0, + "step": 2650 + }, + { + "entropy": 0.17962845370173455, + "epoch": 6.6187845303867405, + "grad_norm": 0.5606568455696106, + "learning_rate": 6.727962422763097e-05, + "loss": 0.12085281372070313, + "mean_token_accuracy": 0.9565140584111214, + "num_tokens": 7296794.0, + "step": 2700 + }, + { + "entropy": 0.1846459148451686, + "epoch": 6.741559238796808, + "grad_norm": 0.5243878364562988, + "learning_rate": 6.302716379757798e-05, + "loss": 0.12253878593444824, + "mean_token_accuracy": 0.9561242800951004, + "num_tokens": 7429156.0, + "step": 2750 + }, + { + "entropy": 0.17984333034604788, + "epoch": 6.864333947206875, + "grad_norm": 0.5876769423484802, + "learning_rate": 5.885777741589364e-05, + "loss": 0.11955853462219239, + "mean_token_accuracy": 0.9572344574332238, + "num_tokens": 7559658.0, + "step": 2800 + }, + { + "entropy": 0.1771458223089576, + "epoch": 6.987108655616943, + "grad_norm": 0.5043931007385254, + "learning_rate": 5.477909360319267e-05, + "loss": 0.11639300346374512, + "mean_token_accuracy": 0.9580946227908135, + "num_tokens": 7699096.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.2669620673997062, + "eval_loss": 0.774162769317627, + "eval_mean_token_accuracy": 0.8417869962964739, + "eval_num_tokens": 7712376.0, + "eval_runtime": 108.7617, + "eval_samples_per_second": 12.835, + "eval_steps_per_second": 1.609, + "step": 2856 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2531777650303386e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ad2c3d366bb2a6896132cf4e21f72c1c718d2a42 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.09831666542701797, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3a38f4363bcd5f78c3cc3db18e364f2fef37d008 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/trainer_state.json @@ -0,0 +1,772 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3264, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6558652856945992, + "epoch": 0.12277470841006753, + "grad_norm": 0.8294193148612976, + "learning_rate": 2.604481765338724e-05, + "loss": 1.5730807495117187, + "mean_token_accuracy": 0.6558220008015633, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.8554385647177696, + "epoch": 0.24554941682013506, + "grad_norm": 1.029894232749939, + "learning_rate": 5.2621162197659935e-05, + "loss": 0.7961322021484375, + "mean_token_accuracy": 0.7755533090233803, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7380270153284073, + "epoch": 0.3683241252302026, + "grad_norm": 0.6096898913383484, + "learning_rate": 7.919750674193263e-05, + "loss": 0.6843100738525391, + "mean_token_accuracy": 0.7998733684420586, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6932123881578446, + "epoch": 0.4910988336402701, + "grad_norm": 0.5934288501739502, + "learning_rate": 0.00010577385128620532, + "loss": 0.6458084106445312, + "mean_token_accuracy": 0.8083393195271492, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6840551143884659, + "epoch": 0.6138735420503376, + "grad_norm": 0.4665544331073761, + "learning_rate": 0.00013235019583047802, + "loss": 0.6333241653442383, + "mean_token_accuracy": 0.8122684139013291, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6649293206632138, + "epoch": 0.7366482504604052, + "grad_norm": 0.47893014550209045, + "learning_rate": 0.00015892654037475069, + "loss": 0.6107040786743164, + "mean_token_accuracy": 0.8165940269827843, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6492742404341698, + "epoch": 0.8594229588704727, + "grad_norm": 0.6059293746948242, + "learning_rate": 0.0001855028849190234, + "loss": 0.597186050415039, + "mean_token_accuracy": 0.8217281407117843, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.642182088047266, + "epoch": 0.9821976672805403, + "grad_norm": 0.3793066143989563, + "learning_rate": 0.0002120792294632961, + "loss": 0.5949800491333008, + "mean_token_accuracy": 0.8229166463017463, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6541631272860936, + "eval_loss": 0.5903413891792297, + "eval_mean_token_accuracy": 0.8252546640804835, + "eval_num_tokens": 1101768.0, + "eval_runtime": 108.9056, + "eval_samples_per_second": 12.818, + "eval_steps_per_second": 1.607, + "step": 408 + }, + { + "entropy": 0.6077755300829253, + "epoch": 1.1031307550644567, + "grad_norm": 0.5028847455978394, + "learning_rate": 0.00021679626884558217, + "loss": 0.5617346954345703, + "mean_token_accuracy": 0.8301527442665875, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5860125370323658, + "epoch": 1.2259054634745243, + "grad_norm": 0.38494572043418884, + "learning_rate": 0.00021653451093163906, + "loss": 0.5437137985229492, + "mean_token_accuracy": 0.8346565261483192, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5945393888652325, + "epoch": 1.3486801718845918, + "grad_norm": 0.5123576521873474, + "learning_rate": 0.00021607496224450087, + "loss": 0.54191650390625, + "mean_token_accuracy": 0.8335164493322372, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.6083934807777405, + "epoch": 1.4714548802946594, + "grad_norm": 0.46661439538002014, + "learning_rate": 0.000215418463597734, + "loss": 0.5550478744506836, + "mean_token_accuracy": 0.8311341696977615, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5862340961396694, + "epoch": 1.5942295887047269, + "grad_norm": 0.49572765827178955, + "learning_rate": 0.00021456621615453177, + "loss": 0.5297146606445312, + "mean_token_accuracy": 0.8360210624337197, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5832360745966434, + "epoch": 1.7170042971147943, + "grad_norm": 0.35520103573799133, + "learning_rate": 0.0002135197792300053, + "loss": 0.5287040328979492, + "mean_token_accuracy": 0.8368027776479721, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5792283065617084, + "epoch": 1.839779005524862, + "grad_norm": 0.3548352122306824, + "learning_rate": 0.00021228106743818178, + "loss": 0.5251237487792969, + "mean_token_accuracy": 0.8396679371595382, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5694268302619457, + "epoch": 1.9625537139349294, + "grad_norm": 0.3476282060146332, + "learning_rate": 0.00021085234718892933, + "loss": 0.5189918899536132, + "mean_token_accuracy": 0.8398650133609772, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5552395452771868, + "eval_loss": 0.5418588519096375, + "eval_mean_token_accuracy": 0.8379638079234532, + "eval_num_tokens": 2203536.0, + "eval_runtime": 108.6649, + "eval_samples_per_second": 12.847, + "eval_steps_per_second": 1.61, + "step": 816 + }, + { + "entropy": 0.5162206395023365, + "epoch": 2.0834868017188457, + "grad_norm": 0.3772931396961212, + "learning_rate": 0.0002092362325412188, + "loss": 0.4619992446899414, + "mean_token_accuracy": 0.8528054483650904, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.5109021583199501, + "epoch": 2.2062615101289134, + "grad_norm": 0.4661090672016144, + "learning_rate": 0.000207435680420309, + "loss": 0.4571444702148437, + "mean_token_accuracy": 0.8552933797240257, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.5012516237795352, + "epoch": 2.329036218538981, + "grad_norm": 0.5193169713020325, + "learning_rate": 0.0002054539852076065, + "loss": 0.45432735443115235, + "mean_token_accuracy": 0.8564087572693825, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.5173932652175427, + "epoch": 2.4518109269490487, + "grad_norm": 0.4666334390640259, + "learning_rate": 0.00020329477271309812, + "loss": 0.4616986083984375, + "mean_token_accuracy": 0.8532203987240792, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.5169526914507151, + "epoch": 2.574585635359116, + "grad_norm": 0.47921115159988403, + "learning_rate": 0.0002009619935413857, + "loss": 0.45737281799316404, + "mean_token_accuracy": 0.8535552659630775, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.50618562489748, + "epoch": 2.6973603437691835, + "grad_norm": 0.3407684862613678, + "learning_rate": 0.00019845991586345935, + "loss": 0.45972068786621095, + "mean_token_accuracy": 0.8532172521948814, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.5158357314765454, + "epoch": 2.820135052179251, + "grad_norm": 0.47901391983032227, + "learning_rate": 0.00019579311760743563, + "loss": 0.46119583129882813, + "mean_token_accuracy": 0.8536588314175606, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.5069115920364857, + "epoch": 2.942909760589319, + "grad_norm": 0.4287651479244232, + "learning_rate": 0.00019296647808254838, + "loss": 0.45447597503662107, + "mean_token_accuracy": 0.856622197329998, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5097703012398311, + "eval_loss": 0.5291272401809692, + "eval_mean_token_accuracy": 0.8438761404582432, + "eval_num_tokens": 3305304.0, + "eval_runtime": 108.6946, + "eval_samples_per_second": 12.843, + "eval_steps_per_second": 1.61, + "step": 1224 + }, + { + "entropy": 0.4642504287250151, + "epoch": 3.063842848373235, + "grad_norm": 0.37649285793304443, + "learning_rate": 0.00018998516905171711, + "loss": 0.4056727600097656, + "mean_token_accuracy": 0.8674883682110588, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.42494223803281783, + "epoch": 3.1866175567833026, + "grad_norm": 0.38774123787879944, + "learning_rate": 0.00018685464526902752, + "loss": 0.36686546325683594, + "mean_token_accuracy": 0.8779315340518952, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.4314767476916313, + "epoch": 3.3093922651933703, + "grad_norm": 0.4668020009994507, + "learning_rate": 0.0001835806344994371, + "loss": 0.37623233795166017, + "mean_token_accuracy": 0.8753074812889099, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.4394826665520668, + "epoch": 3.4321669736034375, + "grad_norm": 0.5372388362884521, + "learning_rate": 0.00018016912703896598, + "loss": 0.3783976364135742, + "mean_token_accuracy": 0.8756387433409691, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.44438183814287185, + "epoch": 3.554941682013505, + "grad_norm": 0.40710341930389404, + "learning_rate": 0.00017662636475454779, + "loss": 0.3865407180786133, + "mean_token_accuracy": 0.8736134466528892, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.43224798239767553, + "epoch": 3.677716390423573, + "grad_norm": 0.48399633169174194, + "learning_rate": 0.0001729588296635937, + "loss": 0.3756423568725586, + "mean_token_accuracy": 0.8760525667667389, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.4290706454217434, + "epoch": 3.80049109883364, + "grad_norm": 0.46181294322013855, + "learning_rate": 0.00016917323207416466, + "loss": 0.37630157470703124, + "mean_token_accuracy": 0.8745916694402694, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.4355049130320549, + "epoch": 3.9232658072437077, + "grad_norm": 0.41401147842407227, + "learning_rate": 0.00016527649830745195, + "loss": 0.38449142456054686, + "mean_token_accuracy": 0.8750512120127678, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.458916278566633, + "eval_loss": 0.533890962600708, + "eval_mean_token_accuracy": 0.8472024192128863, + "eval_num_tokens": 4407072.0, + "eval_runtime": 108.7554, + "eval_samples_per_second": 12.836, + "eval_steps_per_second": 1.609, + "step": 1632 + }, + { + "entropy": 0.38954928343368667, + "epoch": 4.044198895027624, + "grad_norm": 0.4088170826435089, + "learning_rate": 0.0001612757580250288, + "loss": 0.3357394790649414, + "mean_token_accuracy": 0.8872706052010435, + "num_tokens": 4455549.0, + "step": 1650 + }, + { + "entropy": 0.3478683318942785, + "epoch": 4.166973603437691, + "grad_norm": 0.65646892786026, + "learning_rate": 0.00015717833118406024, + "loss": 0.28736536026000975, + "mean_token_accuracy": 0.90216373950243, + "num_tokens": 4584218.0, + "step": 1700 + }, + { + "entropy": 0.3364737436920404, + "epoch": 4.2897483118477595, + "grad_norm": 0.5015673041343689, + "learning_rate": 0.0001529917146443387, + "loss": 0.28391304016113283, + "mean_token_accuracy": 0.9025878021121025, + "num_tokens": 4724425.0, + "step": 1750 + }, + { + "entropy": 0.34254246324300763, + "epoch": 4.412523020257827, + "grad_norm": 0.4893321096897125, + "learning_rate": 0.00014872356845164908, + "loss": 0.28648223876953127, + "mean_token_accuracy": 0.9010315355658531, + "num_tokens": 4861552.0, + "step": 1800 + }, + { + "entropy": 0.35019800525158645, + "epoch": 4.535297728667894, + "grad_norm": 0.5732161998748779, + "learning_rate": 0.0001443817018225613, + "loss": 0.2940273475646973, + "mean_token_accuracy": 0.8994512927532196, + "num_tokens": 4993610.0, + "step": 1850 + }, + { + "entropy": 0.3470254339277744, + "epoch": 4.658072437077962, + "grad_norm": 0.5012240409851074, + "learning_rate": 0.0001399740588562913, + "loss": 0.2919456481933594, + "mean_token_accuracy": 0.8995206660032272, + "num_tokens": 5126291.0, + "step": 1900 + }, + { + "entropy": 0.3445418912172318, + "epoch": 4.780847145488029, + "grad_norm": 0.49685943126678467, + "learning_rate": 0.00013550870399977507, + "loss": 0.2901294136047363, + "mean_token_accuracy": 0.9009958255290985, + "num_tokens": 5266076.0, + "step": 1950 + }, + { + "entropy": 0.3482987853884697, + "epoch": 4.903621853898097, + "grad_norm": 0.46585920453071594, + "learning_rate": 0.00013099380729254763, + "loss": 0.29066293716430663, + "mean_token_accuracy": 0.8992295026779175, + "num_tokens": 5403761.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3964379857267652, + "eval_loss": 0.5762408971786499, + "eval_mean_token_accuracy": 0.8455211687088012, + "eval_num_tokens": 5508840.0, + "eval_runtime": 108.6475, + "eval_samples_per_second": 12.849, + "eval_steps_per_second": 1.611, + "step": 2040 + }, + { + "entropy": 0.33933334491277106, + "epoch": 5.024554941682013, + "grad_norm": 0.718194305896759, + "learning_rate": 0.00012643762941842452, + "loss": 0.2779405975341797, + "mean_token_accuracy": 0.90495823754877, + "num_tokens": 5535743.0, + "step": 2050 + }, + { + "entropy": 0.25700028784573076, + "epoch": 5.147329650092081, + "grad_norm": 0.5866787433624268, + "learning_rate": 0.00012184850659133646, + "loss": 0.19206731796264648, + "mean_token_accuracy": 0.9317478793859482, + "num_tokens": 5675294.0, + "step": 2100 + }, + { + "entropy": 0.26606475934386253, + "epoch": 5.270104358502149, + "grad_norm": 0.5036929845809937, + "learning_rate": 0.00011723483530296943, + "loss": 0.20060159683227538, + "mean_token_accuracy": 0.928264719247818, + "num_tokens": 5808279.0, + "step": 2150 + }, + { + "entropy": 0.2522051434218884, + "epoch": 5.392879066912216, + "grad_norm": 0.5887715220451355, + "learning_rate": 0.00011260505696011844, + "loss": 0.19347431182861327, + "mean_token_accuracy": 0.930549119412899, + "num_tokens": 5948774.0, + "step": 2200 + }, + { + "entropy": 0.26358571454882623, + "epoch": 5.515653775322283, + "grad_norm": 0.5937269330024719, + "learning_rate": 0.00010796764243986134, + "loss": 0.20256309509277343, + "mean_token_accuracy": 0.9277606004476547, + "num_tokens": 6081211.0, + "step": 2250 + }, + { + "entropy": 0.2550286700576544, + "epoch": 5.638428483732351, + "grad_norm": 0.6088381409645081, + "learning_rate": 0.00010333107659081343, + "loss": 0.19648813247680663, + "mean_token_accuracy": 0.9293895545601845, + "num_tokens": 6213793.0, + "step": 2300 + }, + { + "entropy": 0.262065271884203, + "epoch": 5.7612031921424185, + "grad_norm": 0.5331980586051941, + "learning_rate": 9.870384270881798e-05, + "loss": 0.2019593048095703, + "mean_token_accuracy": 0.9276418074965477, + "num_tokens": 6350388.0, + "step": 2350 + }, + { + "entropy": 0.2581966924667358, + "epoch": 5.883977900552486, + "grad_norm": 0.6100393533706665, + "learning_rate": 9.409440701547846e-05, + "loss": 0.20058013916015624, + "mean_token_accuracy": 0.928467052578926, + "num_tokens": 6485960.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.31780434923512596, + "eval_loss": 0.6719555258750916, + "eval_mean_token_accuracy": 0.8434631654194423, + "eval_num_tokens": 6610608.0, + "eval_runtime": 108.6811, + "eval_samples_per_second": 12.845, + "eval_steps_per_second": 1.61, + "step": 2448 + }, + { + "entropy": 0.2578806725855406, + "epoch": 6.004910988336403, + "grad_norm": 0.40365952253341675, + "learning_rate": 8.951120316793048e-05, + "loss": 0.19516365051269532, + "mean_token_accuracy": 0.930728108749777, + "num_tokens": 6616117.0, + "step": 2450 + }, + { + "entropy": 0.18210563194006682, + "epoch": 6.12768569674647, + "grad_norm": 0.5063093304634094, + "learning_rate": 8.496261682819517e-05, + "loss": 0.11744135856628418, + "mean_token_accuracy": 0.9580859485268592, + "num_tokens": 6754569.0, + "step": 2500 + }, + { + "entropy": 0.18104823548346757, + "epoch": 6.250460405156538, + "grad_norm": 0.5599386096000671, + "learning_rate": 8.045697032034715e-05, + "loss": 0.11880653381347656, + "mean_token_accuracy": 0.9568264433741569, + "num_tokens": 6893447.0, + "step": 2550 + }, + { + "entropy": 0.18421781908720733, + "epoch": 6.373235113566605, + "grad_norm": 0.5448400378227234, + "learning_rate": 7.600250740356843e-05, + "loss": 0.11898751258850097, + "mean_token_accuracy": 0.9573441597819329, + "num_tokens": 7025939.0, + "step": 2600 + }, + { + "entropy": 0.18598256688565015, + "epoch": 6.496009821976672, + "grad_norm": 0.49465450644493103, + "learning_rate": 7.16073781889491e-05, + "loss": 0.11917055130004883, + "mean_token_accuracy": 0.9567968577146531, + "num_tokens": 7163978.0, + "step": 2650 + }, + { + "entropy": 0.17962845370173455, + "epoch": 6.6187845303867405, + "grad_norm": 0.5606568455696106, + "learning_rate": 6.727962422763097e-05, + "loss": 0.12085281372070313, + "mean_token_accuracy": 0.9565140584111214, + "num_tokens": 7296794.0, + "step": 2700 + }, + { + "entropy": 0.1846459148451686, + "epoch": 6.741559238796808, + "grad_norm": 0.5243878364562988, + "learning_rate": 6.302716379757798e-05, + "loss": 0.12253878593444824, + "mean_token_accuracy": 0.9561242800951004, + "num_tokens": 7429156.0, + "step": 2750 + }, + { + "entropy": 0.17984333034604788, + "epoch": 6.864333947206875, + "grad_norm": 0.5876769423484802, + "learning_rate": 5.885777741589364e-05, + "loss": 0.11955853462219239, + "mean_token_accuracy": 0.9572344574332238, + "num_tokens": 7559658.0, + "step": 2800 + }, + { + "entropy": 0.1771458223089576, + "epoch": 6.987108655616943, + "grad_norm": 0.5043931007385254, + "learning_rate": 5.477909360319267e-05, + "loss": 0.11639300346374512, + "mean_token_accuracy": 0.9580946227908135, + "num_tokens": 7699096.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.2669620673997062, + "eval_loss": 0.774162769317627, + "eval_mean_token_accuracy": 0.8417869962964739, + "eval_num_tokens": 7712376.0, + "eval_runtime": 108.7617, + "eval_samples_per_second": 12.835, + "eval_steps_per_second": 1.609, + "step": 2856 + }, + { + "entropy": 0.15268251057808774, + "epoch": 7.108041743400859, + "grad_norm": 0.421937495470047, + "learning_rate": 5.079857492607303e-05, + "loss": 0.08690091133117676, + "mean_token_accuracy": 0.9695342624853105, + "num_tokens": 7825131.0, + "step": 2900 + }, + { + "entropy": 0.14063113152980805, + "epoch": 7.230816451810927, + "grad_norm": 0.35511091351509094, + "learning_rate": 4.692350434322603e-05, + "loss": 0.07454932689666748, + "mean_token_accuracy": 0.9723883396387101, + "num_tokens": 7963208.0, + "step": 2950 + }, + { + "entropy": 0.13641462391242384, + "epoch": 7.3535911602209945, + "grad_norm": 0.33150219917297363, + "learning_rate": 4.3160971880165837e-05, + "loss": 0.07405156135559082, + "mean_token_accuracy": 0.9731479465961457, + "num_tokens": 8100530.0, + "step": 3000 + }, + { + "entropy": 0.12723070368170739, + "epoch": 7.476365868631062, + "grad_norm": 0.3763302266597748, + "learning_rate": 3.9517861656959645e-05, + "loss": 0.07087192535400391, + "mean_token_accuracy": 0.9746569180488587, + "num_tokens": 8241781.0, + "step": 3050 + }, + { + "entropy": 0.1335552130267024, + "epoch": 7.59914057704113, + "grad_norm": 0.5557640790939331, + "learning_rate": 3.600083929269262e-05, + "loss": 0.07477219581604004, + "mean_token_accuracy": 0.9728479567170143, + "num_tokens": 8374913.0, + "step": 3100 + }, + { + "entropy": 0.13618831865489484, + "epoch": 7.721915285451197, + "grad_norm": 0.508499801158905, + "learning_rate": 3.261633970971359e-05, + "loss": 0.07407661437988282, + "mean_token_accuracy": 0.9726669636368751, + "num_tokens": 8511662.0, + "step": 3150 + }, + { + "entropy": 0.13547602999955416, + "epoch": 7.844689993861264, + "grad_norm": 0.32131460309028625, + "learning_rate": 2.9370555359974904e-05, + "loss": 0.07703304290771484, + "mean_token_accuracy": 0.972478207051754, + "num_tokens": 8645008.0, + "step": 3200 + }, + { + "entropy": 0.1332347623631358, + "epoch": 7.967464702271332, + "grad_norm": 0.4377666413784027, + "learning_rate": 2.6269424895008482e-05, + "loss": 0.07515048027038575, + "mean_token_accuracy": 0.9730277815461159, + "num_tokens": 8778619.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2319684133359364, + "eval_loss": 0.8868282437324524, + "eval_mean_token_accuracy": 0.8414362386294774, + "eval_num_tokens": 8814144.0, + "eval_runtime": 108.7029, + "eval_samples_per_second": 12.842, + "eval_steps_per_second": 1.61, + "step": 3264 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.43189896749269e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ad2c3d366bb2a6896132cf4e21f72c1c718d2a42 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.09831666542701797, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..cec79348d4b255ce1a94252fc9c845bec83fd9d7 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/trainer_state.json @@ -0,0 +1,863 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3672, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6558652856945992, + "epoch": 0.12277470841006753, + "grad_norm": 0.8294193148612976, + "learning_rate": 2.604481765338724e-05, + "loss": 1.5730807495117187, + "mean_token_accuracy": 0.6558220008015633, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.8554385647177696, + "epoch": 0.24554941682013506, + "grad_norm": 1.029894232749939, + "learning_rate": 5.2621162197659935e-05, + "loss": 0.7961322021484375, + "mean_token_accuracy": 0.7755533090233803, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7380270153284073, + "epoch": 0.3683241252302026, + "grad_norm": 0.6096898913383484, + "learning_rate": 7.919750674193263e-05, + "loss": 0.6843100738525391, + "mean_token_accuracy": 0.7998733684420586, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6932123881578446, + "epoch": 0.4910988336402701, + "grad_norm": 0.5934288501739502, + "learning_rate": 0.00010577385128620532, + "loss": 0.6458084106445312, + "mean_token_accuracy": 0.8083393195271492, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6840551143884659, + "epoch": 0.6138735420503376, + "grad_norm": 0.4665544331073761, + "learning_rate": 0.00013235019583047802, + "loss": 0.6333241653442383, + "mean_token_accuracy": 0.8122684139013291, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6649293206632138, + "epoch": 0.7366482504604052, + "grad_norm": 0.47893014550209045, + "learning_rate": 0.00015892654037475069, + "loss": 0.6107040786743164, + "mean_token_accuracy": 0.8165940269827843, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6492742404341698, + "epoch": 0.8594229588704727, + "grad_norm": 0.6059293746948242, + "learning_rate": 0.0001855028849190234, + "loss": 0.597186050415039, + "mean_token_accuracy": 0.8217281407117843, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.642182088047266, + "epoch": 0.9821976672805403, + "grad_norm": 0.3793066143989563, + "learning_rate": 0.0002120792294632961, + "loss": 0.5949800491333008, + "mean_token_accuracy": 0.8229166463017463, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6541631272860936, + "eval_loss": 0.5903413891792297, + "eval_mean_token_accuracy": 0.8252546640804835, + "eval_num_tokens": 1101768.0, + "eval_runtime": 108.9056, + "eval_samples_per_second": 12.818, + "eval_steps_per_second": 1.607, + "step": 408 + }, + { + "entropy": 0.6077755300829253, + "epoch": 1.1031307550644567, + "grad_norm": 0.5028847455978394, + "learning_rate": 0.00021679626884558217, + "loss": 0.5617346954345703, + "mean_token_accuracy": 0.8301527442665875, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5860125370323658, + "epoch": 1.2259054634745243, + "grad_norm": 0.38494572043418884, + "learning_rate": 0.00021653451093163906, + "loss": 0.5437137985229492, + "mean_token_accuracy": 0.8346565261483192, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5945393888652325, + "epoch": 1.3486801718845918, + "grad_norm": 0.5123576521873474, + "learning_rate": 0.00021607496224450087, + "loss": 0.54191650390625, + "mean_token_accuracy": 0.8335164493322372, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.6083934807777405, + "epoch": 1.4714548802946594, + "grad_norm": 0.46661439538002014, + "learning_rate": 0.000215418463597734, + "loss": 0.5550478744506836, + "mean_token_accuracy": 0.8311341696977615, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5862340961396694, + "epoch": 1.5942295887047269, + "grad_norm": 0.49572765827178955, + "learning_rate": 0.00021456621615453177, + "loss": 0.5297146606445312, + "mean_token_accuracy": 0.8360210624337197, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5832360745966434, + "epoch": 1.7170042971147943, + "grad_norm": 0.35520103573799133, + "learning_rate": 0.0002135197792300053, + "loss": 0.5287040328979492, + "mean_token_accuracy": 0.8368027776479721, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5792283065617084, + "epoch": 1.839779005524862, + "grad_norm": 0.3548352122306824, + "learning_rate": 0.00021228106743818178, + "loss": 0.5251237487792969, + "mean_token_accuracy": 0.8396679371595382, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5694268302619457, + "epoch": 1.9625537139349294, + "grad_norm": 0.3476282060146332, + "learning_rate": 0.00021085234718892933, + "loss": 0.5189918899536132, + "mean_token_accuracy": 0.8398650133609772, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5552395452771868, + "eval_loss": 0.5418588519096375, + "eval_mean_token_accuracy": 0.8379638079234532, + "eval_num_tokens": 2203536.0, + "eval_runtime": 108.6649, + "eval_samples_per_second": 12.847, + "eval_steps_per_second": 1.61, + "step": 816 + }, + { + "entropy": 0.5162206395023365, + "epoch": 2.0834868017188457, + "grad_norm": 0.3772931396961212, + "learning_rate": 0.0002092362325412188, + "loss": 0.4619992446899414, + "mean_token_accuracy": 0.8528054483650904, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.5109021583199501, + "epoch": 2.2062615101289134, + "grad_norm": 0.4661090672016144, + "learning_rate": 0.000207435680420309, + "loss": 0.4571444702148437, + "mean_token_accuracy": 0.8552933797240257, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.5012516237795352, + "epoch": 2.329036218538981, + "grad_norm": 0.5193169713020325, + "learning_rate": 0.0002054539852076065, + "loss": 0.45432735443115235, + "mean_token_accuracy": 0.8564087572693825, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.5173932652175427, + "epoch": 2.4518109269490487, + "grad_norm": 0.4666334390640259, + "learning_rate": 0.00020329477271309812, + "loss": 0.4616986083984375, + "mean_token_accuracy": 0.8532203987240792, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.5169526914507151, + "epoch": 2.574585635359116, + "grad_norm": 0.47921115159988403, + "learning_rate": 0.0002009619935413857, + "loss": 0.45737281799316404, + "mean_token_accuracy": 0.8535552659630775, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.50618562489748, + "epoch": 2.6973603437691835, + "grad_norm": 0.3407684862613678, + "learning_rate": 0.00019845991586345935, + "loss": 0.45972068786621095, + "mean_token_accuracy": 0.8532172521948814, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.5158357314765454, + "epoch": 2.820135052179251, + "grad_norm": 0.47901391983032227, + "learning_rate": 0.00019579311760743563, + "loss": 0.46119583129882813, + "mean_token_accuracy": 0.8536588314175606, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.5069115920364857, + "epoch": 2.942909760589319, + "grad_norm": 0.4287651479244232, + "learning_rate": 0.00019296647808254838, + "loss": 0.45447597503662107, + "mean_token_accuracy": 0.856622197329998, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5097703012398311, + "eval_loss": 0.5291272401809692, + "eval_mean_token_accuracy": 0.8438761404582432, + "eval_num_tokens": 3305304.0, + "eval_runtime": 108.6946, + "eval_samples_per_second": 12.843, + "eval_steps_per_second": 1.61, + "step": 1224 + }, + { + "entropy": 0.4642504287250151, + "epoch": 3.063842848373235, + "grad_norm": 0.37649285793304443, + "learning_rate": 0.00018998516905171711, + "loss": 0.4056727600097656, + "mean_token_accuracy": 0.8674883682110588, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.42494223803281783, + "epoch": 3.1866175567833026, + "grad_norm": 0.38774123787879944, + "learning_rate": 0.00018685464526902752, + "loss": 0.36686546325683594, + "mean_token_accuracy": 0.8779315340518952, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.4314767476916313, + "epoch": 3.3093922651933703, + "grad_norm": 0.4668020009994507, + "learning_rate": 0.0001835806344994371, + "loss": 0.37623233795166017, + "mean_token_accuracy": 0.8753074812889099, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.4394826665520668, + "epoch": 3.4321669736034375, + "grad_norm": 0.5372388362884521, + "learning_rate": 0.00018016912703896598, + "loss": 0.3783976364135742, + "mean_token_accuracy": 0.8756387433409691, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.44438183814287185, + "epoch": 3.554941682013505, + "grad_norm": 0.40710341930389404, + "learning_rate": 0.00017662636475454779, + "loss": 0.3865407180786133, + "mean_token_accuracy": 0.8736134466528892, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.43224798239767553, + "epoch": 3.677716390423573, + "grad_norm": 0.48399633169174194, + "learning_rate": 0.0001729588296635937, + "loss": 0.3756423568725586, + "mean_token_accuracy": 0.8760525667667389, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.4290706454217434, + "epoch": 3.80049109883364, + "grad_norm": 0.46181294322013855, + "learning_rate": 0.00016917323207416466, + "loss": 0.37630157470703124, + "mean_token_accuracy": 0.8745916694402694, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.4355049130320549, + "epoch": 3.9232658072437077, + "grad_norm": 0.41401147842407227, + "learning_rate": 0.00016527649830745195, + "loss": 0.38449142456054686, + "mean_token_accuracy": 0.8750512120127678, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.458916278566633, + "eval_loss": 0.533890962600708, + "eval_mean_token_accuracy": 0.8472024192128863, + "eval_num_tokens": 4407072.0, + "eval_runtime": 108.7554, + "eval_samples_per_second": 12.836, + "eval_steps_per_second": 1.609, + "step": 1632 + }, + { + "entropy": 0.38954928343368667, + "epoch": 4.044198895027624, + "grad_norm": 0.4088170826435089, + "learning_rate": 0.0001612757580250288, + "loss": 0.3357394790649414, + "mean_token_accuracy": 0.8872706052010435, + "num_tokens": 4455549.0, + "step": 1650 + }, + { + "entropy": 0.3478683318942785, + "epoch": 4.166973603437691, + "grad_norm": 0.65646892786026, + "learning_rate": 0.00015717833118406024, + "loss": 0.28736536026000975, + "mean_token_accuracy": 0.90216373950243, + "num_tokens": 4584218.0, + "step": 1700 + }, + { + "entropy": 0.3364737436920404, + "epoch": 4.2897483118477595, + "grad_norm": 0.5015673041343689, + "learning_rate": 0.0001529917146443387, + "loss": 0.28391304016113283, + "mean_token_accuracy": 0.9025878021121025, + "num_tokens": 4724425.0, + "step": 1750 + }, + { + "entropy": 0.34254246324300763, + "epoch": 4.412523020257827, + "grad_norm": 0.4893321096897125, + "learning_rate": 0.00014872356845164908, + "loss": 0.28648223876953127, + "mean_token_accuracy": 0.9010315355658531, + "num_tokens": 4861552.0, + "step": 1800 + }, + { + "entropy": 0.35019800525158645, + "epoch": 4.535297728667894, + "grad_norm": 0.5732161998748779, + "learning_rate": 0.0001443817018225613, + "loss": 0.2940273475646973, + "mean_token_accuracy": 0.8994512927532196, + "num_tokens": 4993610.0, + "step": 1850 + }, + { + "entropy": 0.3470254339277744, + "epoch": 4.658072437077962, + "grad_norm": 0.5012240409851074, + "learning_rate": 0.0001399740588562913, + "loss": 0.2919456481933594, + "mean_token_accuracy": 0.8995206660032272, + "num_tokens": 5126291.0, + "step": 1900 + }, + { + "entropy": 0.3445418912172318, + "epoch": 4.780847145488029, + "grad_norm": 0.49685943126678467, + "learning_rate": 0.00013550870399977507, + "loss": 0.2901294136047363, + "mean_token_accuracy": 0.9009958255290985, + "num_tokens": 5266076.0, + "step": 1950 + }, + { + "entropy": 0.3482987853884697, + "epoch": 4.903621853898097, + "grad_norm": 0.46585920453071594, + "learning_rate": 0.00013099380729254763, + "loss": 0.29066293716430663, + "mean_token_accuracy": 0.8992295026779175, + "num_tokens": 5403761.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3964379857267652, + "eval_loss": 0.5762408971786499, + "eval_mean_token_accuracy": 0.8455211687088012, + "eval_num_tokens": 5508840.0, + "eval_runtime": 108.6475, + "eval_samples_per_second": 12.849, + "eval_steps_per_second": 1.611, + "step": 2040 + }, + { + "entropy": 0.33933334491277106, + "epoch": 5.024554941682013, + "grad_norm": 0.718194305896759, + "learning_rate": 0.00012643762941842452, + "loss": 0.2779405975341797, + "mean_token_accuracy": 0.90495823754877, + "num_tokens": 5535743.0, + "step": 2050 + }, + { + "entropy": 0.25700028784573076, + "epoch": 5.147329650092081, + "grad_norm": 0.5866787433624268, + "learning_rate": 0.00012184850659133646, + "loss": 0.19206731796264648, + "mean_token_accuracy": 0.9317478793859482, + "num_tokens": 5675294.0, + "step": 2100 + }, + { + "entropy": 0.26606475934386253, + "epoch": 5.270104358502149, + "grad_norm": 0.5036929845809937, + "learning_rate": 0.00011723483530296943, + "loss": 0.20060159683227538, + "mean_token_accuracy": 0.928264719247818, + "num_tokens": 5808279.0, + "step": 2150 + }, + { + "entropy": 0.2522051434218884, + "epoch": 5.392879066912216, + "grad_norm": 0.5887715220451355, + "learning_rate": 0.00011260505696011844, + "loss": 0.19347431182861327, + "mean_token_accuracy": 0.930549119412899, + "num_tokens": 5948774.0, + "step": 2200 + }, + { + "entropy": 0.26358571454882623, + "epoch": 5.515653775322283, + "grad_norm": 0.5937269330024719, + "learning_rate": 0.00010796764243986134, + "loss": 0.20256309509277343, + "mean_token_accuracy": 0.9277606004476547, + "num_tokens": 6081211.0, + "step": 2250 + }, + { + "entropy": 0.2550286700576544, + "epoch": 5.638428483732351, + "grad_norm": 0.6088381409645081, + "learning_rate": 0.00010333107659081343, + "loss": 0.19648813247680663, + "mean_token_accuracy": 0.9293895545601845, + "num_tokens": 6213793.0, + "step": 2300 + }, + { + "entropy": 0.262065271884203, + "epoch": 5.7612031921424185, + "grad_norm": 0.5331980586051941, + "learning_rate": 9.870384270881798e-05, + "loss": 0.2019593048095703, + "mean_token_accuracy": 0.9276418074965477, + "num_tokens": 6350388.0, + "step": 2350 + }, + { + "entropy": 0.2581966924667358, + "epoch": 5.883977900552486, + "grad_norm": 0.6100393533706665, + "learning_rate": 9.409440701547846e-05, + "loss": 0.20058013916015624, + "mean_token_accuracy": 0.928467052578926, + "num_tokens": 6485960.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.31780434923512596, + "eval_loss": 0.6719555258750916, + "eval_mean_token_accuracy": 0.8434631654194423, + "eval_num_tokens": 6610608.0, + "eval_runtime": 108.6811, + "eval_samples_per_second": 12.845, + "eval_steps_per_second": 1.61, + "step": 2448 + }, + { + "entropy": 0.2578806725855406, + "epoch": 6.004910988336403, + "grad_norm": 0.40365952253341675, + "learning_rate": 8.951120316793048e-05, + "loss": 0.19516365051269532, + "mean_token_accuracy": 0.930728108749777, + "num_tokens": 6616117.0, + "step": 2450 + }, + { + "entropy": 0.18210563194006682, + "epoch": 6.12768569674647, + "grad_norm": 0.5063093304634094, + "learning_rate": 8.496261682819517e-05, + "loss": 0.11744135856628418, + "mean_token_accuracy": 0.9580859485268592, + "num_tokens": 6754569.0, + "step": 2500 + }, + { + "entropy": 0.18104823548346757, + "epoch": 6.250460405156538, + "grad_norm": 0.5599386096000671, + "learning_rate": 8.045697032034715e-05, + "loss": 0.11880653381347656, + "mean_token_accuracy": 0.9568264433741569, + "num_tokens": 6893447.0, + "step": 2550 + }, + { + "entropy": 0.18421781908720733, + "epoch": 6.373235113566605, + "grad_norm": 0.5448400378227234, + "learning_rate": 7.600250740356843e-05, + "loss": 0.11898751258850097, + "mean_token_accuracy": 0.9573441597819329, + "num_tokens": 7025939.0, + "step": 2600 + }, + { + "entropy": 0.18598256688565015, + "epoch": 6.496009821976672, + "grad_norm": 0.49465450644493103, + "learning_rate": 7.16073781889491e-05, + "loss": 0.11917055130004883, + "mean_token_accuracy": 0.9567968577146531, + "num_tokens": 7163978.0, + "step": 2650 + }, + { + "entropy": 0.17962845370173455, + "epoch": 6.6187845303867405, + "grad_norm": 0.5606568455696106, + "learning_rate": 6.727962422763097e-05, + "loss": 0.12085281372070313, + "mean_token_accuracy": 0.9565140584111214, + "num_tokens": 7296794.0, + "step": 2700 + }, + { + "entropy": 0.1846459148451686, + "epoch": 6.741559238796808, + "grad_norm": 0.5243878364562988, + "learning_rate": 6.302716379757798e-05, + "loss": 0.12253878593444824, + "mean_token_accuracy": 0.9561242800951004, + "num_tokens": 7429156.0, + "step": 2750 + }, + { + "entropy": 0.17984333034604788, + "epoch": 6.864333947206875, + "grad_norm": 0.5876769423484802, + "learning_rate": 5.885777741589364e-05, + "loss": 0.11955853462219239, + "mean_token_accuracy": 0.9572344574332238, + "num_tokens": 7559658.0, + "step": 2800 + }, + { + "entropy": 0.1771458223089576, + "epoch": 6.987108655616943, + "grad_norm": 0.5043931007385254, + "learning_rate": 5.477909360319267e-05, + "loss": 0.11639300346374512, + "mean_token_accuracy": 0.9580946227908135, + "num_tokens": 7699096.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.2669620673997062, + "eval_loss": 0.774162769317627, + "eval_mean_token_accuracy": 0.8417869962964739, + "eval_num_tokens": 7712376.0, + "eval_runtime": 108.7617, + "eval_samples_per_second": 12.835, + "eval_steps_per_second": 1.609, + "step": 2856 + }, + { + "entropy": 0.15268251057808774, + "epoch": 7.108041743400859, + "grad_norm": 0.421937495470047, + "learning_rate": 5.079857492607303e-05, + "loss": 0.08690091133117676, + "mean_token_accuracy": 0.9695342624853105, + "num_tokens": 7825131.0, + "step": 2900 + }, + { + "entropy": 0.14063113152980805, + "epoch": 7.230816451810927, + "grad_norm": 0.35511091351509094, + "learning_rate": 4.692350434322603e-05, + "loss": 0.07454932689666748, + "mean_token_accuracy": 0.9723883396387101, + "num_tokens": 7963208.0, + "step": 2950 + }, + { + "entropy": 0.13641462391242384, + "epoch": 7.3535911602209945, + "grad_norm": 0.33150219917297363, + "learning_rate": 4.3160971880165837e-05, + "loss": 0.07405156135559082, + "mean_token_accuracy": 0.9731479465961457, + "num_tokens": 8100530.0, + "step": 3000 + }, + { + "entropy": 0.12723070368170739, + "epoch": 7.476365868631062, + "grad_norm": 0.3763302266597748, + "learning_rate": 3.9517861656959645e-05, + "loss": 0.07087192535400391, + "mean_token_accuracy": 0.9746569180488587, + "num_tokens": 8241781.0, + "step": 3050 + }, + { + "entropy": 0.1335552130267024, + "epoch": 7.59914057704113, + "grad_norm": 0.5557640790939331, + "learning_rate": 3.600083929269262e-05, + "loss": 0.07477219581604004, + "mean_token_accuracy": 0.9728479567170143, + "num_tokens": 8374913.0, + "step": 3100 + }, + { + "entropy": 0.13618831865489484, + "epoch": 7.721915285451197, + "grad_norm": 0.508499801158905, + "learning_rate": 3.261633970971359e-05, + "loss": 0.07407661437988282, + "mean_token_accuracy": 0.9726669636368751, + "num_tokens": 8511662.0, + "step": 3150 + }, + { + "entropy": 0.13547602999955416, + "epoch": 7.844689993861264, + "grad_norm": 0.32131460309028625, + "learning_rate": 2.9370555359974904e-05, + "loss": 0.07703304290771484, + "mean_token_accuracy": 0.972478207051754, + "num_tokens": 8645008.0, + "step": 3200 + }, + { + "entropy": 0.1332347623631358, + "epoch": 7.967464702271332, + "grad_norm": 0.4377666413784027, + "learning_rate": 2.6269424895008482e-05, + "loss": 0.07515048027038575, + "mean_token_accuracy": 0.9730277815461159, + "num_tokens": 8778619.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2319684133359364, + "eval_loss": 0.8868282437324524, + "eval_mean_token_accuracy": 0.8414362386294774, + "eval_num_tokens": 8814144.0, + "eval_runtime": 108.7029, + "eval_samples_per_second": 12.842, + "eval_steps_per_second": 1.61, + "step": 3264 + }, + { + "entropy": 0.12603538889948487, + "epoch": 8.088397790055248, + "grad_norm": 0.34813445806503296, + "learning_rate": 2.3318622300268085e-05, + "loss": 0.06378498077392578, + "mean_token_accuracy": 0.9763191284867108, + "num_tokens": 8907693.0, + "step": 3300 + }, + { + "entropy": 0.11884484702721237, + "epoch": 8.211172498465316, + "grad_norm": 0.22056937217712402, + "learning_rate": 2.0523546513718086e-05, + "loss": 0.059115543365478515, + "mean_token_accuracy": 0.9775407165288925, + "num_tokens": 9041538.0, + "step": 3350 + }, + { + "entropy": 0.1213479395210743, + "epoch": 8.333947206875383, + "grad_norm": 0.30439141392707825, + "learning_rate": 1.7889311547662764e-05, + "loss": 0.060338306427001956, + "mean_token_accuracy": 0.9764804869890213, + "num_tokens": 9173381.0, + "step": 3400 + }, + { + "entropy": 0.12034013645723463, + "epoch": 8.456721915285451, + "grad_norm": 0.2019958645105362, + "learning_rate": 1.5420737131890332e-05, + "loss": 0.05998798847198486, + "mean_token_accuracy": 0.9769419169425965, + "num_tokens": 9306859.0, + "step": 3450 + }, + { + "entropy": 0.11213825253769755, + "epoch": 8.579496623695519, + "grad_norm": 0.21801051497459412, + "learning_rate": 1.3122339895251194e-05, + "loss": 0.05649502277374267, + "mean_token_accuracy": 0.9788470858335495, + "num_tokens": 9451826.0, + "step": 3500 + }, + { + "entropy": 0.11963866963982582, + "epoch": 8.702271332105585, + "grad_norm": 0.31590062379837036, + "learning_rate": 1.0998325101804684e-05, + "loss": 0.05895202159881592, + "mean_token_accuracy": 0.9766810277104377, + "num_tokens": 9590049.0, + "step": 3550 + }, + { + "entropy": 0.11513572301715612, + "epoch": 8.825046040515653, + "grad_norm": 0.18179665505886078, + "learning_rate": 9.052578956654935e-06, + "loss": 0.05937095642089844, + "mean_token_accuracy": 0.9773054680228234, + "num_tokens": 9727075.0, + "step": 3600 + }, + { + "entropy": 0.11545982209965587, + "epoch": 8.947820748925722, + "grad_norm": 0.28307580947875977, + "learning_rate": 7.28866149555317e-06, + "loss": 0.0594486665725708, + "mean_token_accuracy": 0.9774915012717247, + "num_tokens": 9862022.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.21108701620783124, + "eval_loss": 0.9910867810249329, + "eval_mean_token_accuracy": 0.8417104813030788, + "eval_num_tokens": 9915912.0, + "eval_runtime": 108.7188, + "eval_samples_per_second": 12.84, + "eval_steps_per_second": 1.61, + "step": 3672 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6105429116327936e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ad2c3d366bb2a6896132cf4e21f72c1c718d2a42 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.09831666542701797, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..10052297c75090f0bae097f02e93608a6b769928 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 408, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6558652856945992, + "epoch": 0.12277470841006753, + "grad_norm": 0.8294193148612976, + "learning_rate": 2.604481765338724e-05, + "loss": 1.5730807495117187, + "mean_token_accuracy": 0.6558220008015633, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.8554385647177696, + "epoch": 0.24554941682013506, + "grad_norm": 1.029894232749939, + "learning_rate": 5.2621162197659935e-05, + "loss": 0.7961322021484375, + "mean_token_accuracy": 0.7755533090233803, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7380270153284073, + "epoch": 0.3683241252302026, + "grad_norm": 0.6096898913383484, + "learning_rate": 7.919750674193263e-05, + "loss": 0.6843100738525391, + "mean_token_accuracy": 0.7998733684420586, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6932123881578446, + "epoch": 0.4910988336402701, + "grad_norm": 0.5934288501739502, + "learning_rate": 0.00010577385128620532, + "loss": 0.6458084106445312, + "mean_token_accuracy": 0.8083393195271492, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6840551143884659, + "epoch": 0.6138735420503376, + "grad_norm": 0.4665544331073761, + "learning_rate": 0.00013235019583047802, + "loss": 0.6333241653442383, + "mean_token_accuracy": 0.8122684139013291, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6649293206632138, + "epoch": 0.7366482504604052, + "grad_norm": 0.47893014550209045, + "learning_rate": 0.00015892654037475069, + "loss": 0.6107040786743164, + "mean_token_accuracy": 0.8165940269827843, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6492742404341698, + "epoch": 0.8594229588704727, + "grad_norm": 0.6059293746948242, + "learning_rate": 0.0001855028849190234, + "loss": 0.597186050415039, + "mean_token_accuracy": 0.8217281407117843, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.642182088047266, + "epoch": 0.9821976672805403, + "grad_norm": 0.3793066143989563, + "learning_rate": 0.0002120792294632961, + "loss": 0.5949800491333008, + "mean_token_accuracy": 0.8229166463017463, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6541631272860936, + "eval_loss": 0.5903413891792297, + "eval_mean_token_accuracy": 0.8252546640804835, + "eval_num_tokens": 1101768.0, + "eval_runtime": 108.9056, + "eval_samples_per_second": 12.818, + "eval_steps_per_second": 1.607, + "step": 408 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.7928615394378752e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ad2c3d366bb2a6896132cf4e21f72c1c718d2a42 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.09831666542701797, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..49fa5dbd867521ea599c3362c42e0c183b4f29d5 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/trainer_state.json @@ -0,0 +1,954 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4080, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6558652856945992, + "epoch": 0.12277470841006753, + "grad_norm": 0.8294193148612976, + "learning_rate": 2.604481765338724e-05, + "loss": 1.5730807495117187, + "mean_token_accuracy": 0.6558220008015633, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.8554385647177696, + "epoch": 0.24554941682013506, + "grad_norm": 1.029894232749939, + "learning_rate": 5.2621162197659935e-05, + "loss": 0.7961322021484375, + "mean_token_accuracy": 0.7755533090233803, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7380270153284073, + "epoch": 0.3683241252302026, + "grad_norm": 0.6096898913383484, + "learning_rate": 7.919750674193263e-05, + "loss": 0.6843100738525391, + "mean_token_accuracy": 0.7998733684420586, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6932123881578446, + "epoch": 0.4910988336402701, + "grad_norm": 0.5934288501739502, + "learning_rate": 0.00010577385128620532, + "loss": 0.6458084106445312, + "mean_token_accuracy": 0.8083393195271492, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6840551143884659, + "epoch": 0.6138735420503376, + "grad_norm": 0.4665544331073761, + "learning_rate": 0.00013235019583047802, + "loss": 0.6333241653442383, + "mean_token_accuracy": 0.8122684139013291, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6649293206632138, + "epoch": 0.7366482504604052, + "grad_norm": 0.47893014550209045, + "learning_rate": 0.00015892654037475069, + "loss": 0.6107040786743164, + "mean_token_accuracy": 0.8165940269827843, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6492742404341698, + "epoch": 0.8594229588704727, + "grad_norm": 0.6059293746948242, + "learning_rate": 0.0001855028849190234, + "loss": 0.597186050415039, + "mean_token_accuracy": 0.8217281407117843, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.642182088047266, + "epoch": 0.9821976672805403, + "grad_norm": 0.3793066143989563, + "learning_rate": 0.0002120792294632961, + "loss": 0.5949800491333008, + "mean_token_accuracy": 0.8229166463017463, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6541631272860936, + "eval_loss": 0.5903413891792297, + "eval_mean_token_accuracy": 0.8252546640804835, + "eval_num_tokens": 1101768.0, + "eval_runtime": 108.9056, + "eval_samples_per_second": 12.818, + "eval_steps_per_second": 1.607, + "step": 408 + }, + { + "entropy": 0.6077755300829253, + "epoch": 1.1031307550644567, + "grad_norm": 0.5028847455978394, + "learning_rate": 0.00021679626884558217, + "loss": 0.5617346954345703, + "mean_token_accuracy": 0.8301527442665875, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5860125370323658, + "epoch": 1.2259054634745243, + "grad_norm": 0.38494572043418884, + "learning_rate": 0.00021653451093163906, + "loss": 0.5437137985229492, + "mean_token_accuracy": 0.8346565261483192, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5945393888652325, + "epoch": 1.3486801718845918, + "grad_norm": 0.5123576521873474, + "learning_rate": 0.00021607496224450087, + "loss": 0.54191650390625, + "mean_token_accuracy": 0.8335164493322372, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.6083934807777405, + "epoch": 1.4714548802946594, + "grad_norm": 0.46661439538002014, + "learning_rate": 0.000215418463597734, + "loss": 0.5550478744506836, + "mean_token_accuracy": 0.8311341696977615, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5862340961396694, + "epoch": 1.5942295887047269, + "grad_norm": 0.49572765827178955, + "learning_rate": 0.00021456621615453177, + "loss": 0.5297146606445312, + "mean_token_accuracy": 0.8360210624337197, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5832360745966434, + "epoch": 1.7170042971147943, + "grad_norm": 0.35520103573799133, + "learning_rate": 0.0002135197792300053, + "loss": 0.5287040328979492, + "mean_token_accuracy": 0.8368027776479721, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5792283065617084, + "epoch": 1.839779005524862, + "grad_norm": 0.3548352122306824, + "learning_rate": 0.00021228106743818178, + "loss": 0.5251237487792969, + "mean_token_accuracy": 0.8396679371595382, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5694268302619457, + "epoch": 1.9625537139349294, + "grad_norm": 0.3476282060146332, + "learning_rate": 0.00021085234718892933, + "loss": 0.5189918899536132, + "mean_token_accuracy": 0.8398650133609772, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5552395452771868, + "eval_loss": 0.5418588519096375, + "eval_mean_token_accuracy": 0.8379638079234532, + "eval_num_tokens": 2203536.0, + "eval_runtime": 108.6649, + "eval_samples_per_second": 12.847, + "eval_steps_per_second": 1.61, + "step": 816 + }, + { + "entropy": 0.5162206395023365, + "epoch": 2.0834868017188457, + "grad_norm": 0.3772931396961212, + "learning_rate": 0.0002092362325412188, + "loss": 0.4619992446899414, + "mean_token_accuracy": 0.8528054483650904, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.5109021583199501, + "epoch": 2.2062615101289134, + "grad_norm": 0.4661090672016144, + "learning_rate": 0.000207435680420309, + "loss": 0.4571444702148437, + "mean_token_accuracy": 0.8552933797240257, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.5012516237795352, + "epoch": 2.329036218538981, + "grad_norm": 0.5193169713020325, + "learning_rate": 0.0002054539852076065, + "loss": 0.45432735443115235, + "mean_token_accuracy": 0.8564087572693825, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.5173932652175427, + "epoch": 2.4518109269490487, + "grad_norm": 0.4666334390640259, + "learning_rate": 0.00020329477271309812, + "loss": 0.4616986083984375, + "mean_token_accuracy": 0.8532203987240792, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.5169526914507151, + "epoch": 2.574585635359116, + "grad_norm": 0.47921115159988403, + "learning_rate": 0.0002009619935413857, + "loss": 0.45737281799316404, + "mean_token_accuracy": 0.8535552659630775, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.50618562489748, + "epoch": 2.6973603437691835, + "grad_norm": 0.3407684862613678, + "learning_rate": 0.00019845991586345935, + "loss": 0.45972068786621095, + "mean_token_accuracy": 0.8532172521948814, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.5158357314765454, + "epoch": 2.820135052179251, + "grad_norm": 0.47901391983032227, + "learning_rate": 0.00019579311760743563, + "loss": 0.46119583129882813, + "mean_token_accuracy": 0.8536588314175606, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.5069115920364857, + "epoch": 2.942909760589319, + "grad_norm": 0.4287651479244232, + "learning_rate": 0.00019296647808254838, + "loss": 0.45447597503662107, + "mean_token_accuracy": 0.856622197329998, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5097703012398311, + "eval_loss": 0.5291272401809692, + "eval_mean_token_accuracy": 0.8438761404582432, + "eval_num_tokens": 3305304.0, + "eval_runtime": 108.6946, + "eval_samples_per_second": 12.843, + "eval_steps_per_second": 1.61, + "step": 1224 + }, + { + "entropy": 0.4642504287250151, + "epoch": 3.063842848373235, + "grad_norm": 0.37649285793304443, + "learning_rate": 0.00018998516905171711, + "loss": 0.4056727600097656, + "mean_token_accuracy": 0.8674883682110588, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.42494223803281783, + "epoch": 3.1866175567833026, + "grad_norm": 0.38774123787879944, + "learning_rate": 0.00018685464526902752, + "loss": 0.36686546325683594, + "mean_token_accuracy": 0.8779315340518952, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.4314767476916313, + "epoch": 3.3093922651933703, + "grad_norm": 0.4668020009994507, + "learning_rate": 0.0001835806344994371, + "loss": 0.37623233795166017, + "mean_token_accuracy": 0.8753074812889099, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.4394826665520668, + "epoch": 3.4321669736034375, + "grad_norm": 0.5372388362884521, + "learning_rate": 0.00018016912703896598, + "loss": 0.3783976364135742, + "mean_token_accuracy": 0.8756387433409691, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.44438183814287185, + "epoch": 3.554941682013505, + "grad_norm": 0.40710341930389404, + "learning_rate": 0.00017662636475454779, + "loss": 0.3865407180786133, + "mean_token_accuracy": 0.8736134466528892, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.43224798239767553, + "epoch": 3.677716390423573, + "grad_norm": 0.48399633169174194, + "learning_rate": 0.0001729588296635937, + "loss": 0.3756423568725586, + "mean_token_accuracy": 0.8760525667667389, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.4290706454217434, + "epoch": 3.80049109883364, + "grad_norm": 0.46181294322013855, + "learning_rate": 0.00016917323207416466, + "loss": 0.37630157470703124, + "mean_token_accuracy": 0.8745916694402694, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.4355049130320549, + "epoch": 3.9232658072437077, + "grad_norm": 0.41401147842407227, + "learning_rate": 0.00016527649830745195, + "loss": 0.38449142456054686, + "mean_token_accuracy": 0.8750512120127678, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.458916278566633, + "eval_loss": 0.533890962600708, + "eval_mean_token_accuracy": 0.8472024192128863, + "eval_num_tokens": 4407072.0, + "eval_runtime": 108.7554, + "eval_samples_per_second": 12.836, + "eval_steps_per_second": 1.609, + "step": 1632 + }, + { + "entropy": 0.38954928343368667, + "epoch": 4.044198895027624, + "grad_norm": 0.4088170826435089, + "learning_rate": 0.0001612757580250288, + "loss": 0.3357394790649414, + "mean_token_accuracy": 0.8872706052010435, + "num_tokens": 4455549.0, + "step": 1650 + }, + { + "entropy": 0.3478683318942785, + "epoch": 4.166973603437691, + "grad_norm": 0.65646892786026, + "learning_rate": 0.00015717833118406024, + "loss": 0.28736536026000975, + "mean_token_accuracy": 0.90216373950243, + "num_tokens": 4584218.0, + "step": 1700 + }, + { + "entropy": 0.3364737436920404, + "epoch": 4.2897483118477595, + "grad_norm": 0.5015673041343689, + "learning_rate": 0.0001529917146443387, + "loss": 0.28391304016113283, + "mean_token_accuracy": 0.9025878021121025, + "num_tokens": 4724425.0, + "step": 1750 + }, + { + "entropy": 0.34254246324300763, + "epoch": 4.412523020257827, + "grad_norm": 0.4893321096897125, + "learning_rate": 0.00014872356845164908, + "loss": 0.28648223876953127, + "mean_token_accuracy": 0.9010315355658531, + "num_tokens": 4861552.0, + "step": 1800 + }, + { + "entropy": 0.35019800525158645, + "epoch": 4.535297728667894, + "grad_norm": 0.5732161998748779, + "learning_rate": 0.0001443817018225613, + "loss": 0.2940273475646973, + "mean_token_accuracy": 0.8994512927532196, + "num_tokens": 4993610.0, + "step": 1850 + }, + { + "entropy": 0.3470254339277744, + "epoch": 4.658072437077962, + "grad_norm": 0.5012240409851074, + "learning_rate": 0.0001399740588562913, + "loss": 0.2919456481933594, + "mean_token_accuracy": 0.8995206660032272, + "num_tokens": 5126291.0, + "step": 1900 + }, + { + "entropy": 0.3445418912172318, + "epoch": 4.780847145488029, + "grad_norm": 0.49685943126678467, + "learning_rate": 0.00013550870399977507, + "loss": 0.2901294136047363, + "mean_token_accuracy": 0.9009958255290985, + "num_tokens": 5266076.0, + "step": 1950 + }, + { + "entropy": 0.3482987853884697, + "epoch": 4.903621853898097, + "grad_norm": 0.46585920453071594, + "learning_rate": 0.00013099380729254763, + "loss": 0.29066293716430663, + "mean_token_accuracy": 0.8992295026779175, + "num_tokens": 5403761.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3964379857267652, + "eval_loss": 0.5762408971786499, + "eval_mean_token_accuracy": 0.8455211687088012, + "eval_num_tokens": 5508840.0, + "eval_runtime": 108.6475, + "eval_samples_per_second": 12.849, + "eval_steps_per_second": 1.611, + "step": 2040 + }, + { + "entropy": 0.33933334491277106, + "epoch": 5.024554941682013, + "grad_norm": 0.718194305896759, + "learning_rate": 0.00012643762941842452, + "loss": 0.2779405975341797, + "mean_token_accuracy": 0.90495823754877, + "num_tokens": 5535743.0, + "step": 2050 + }, + { + "entropy": 0.25700028784573076, + "epoch": 5.147329650092081, + "grad_norm": 0.5866787433624268, + "learning_rate": 0.00012184850659133646, + "loss": 0.19206731796264648, + "mean_token_accuracy": 0.9317478793859482, + "num_tokens": 5675294.0, + "step": 2100 + }, + { + "entropy": 0.26606475934386253, + "epoch": 5.270104358502149, + "grad_norm": 0.5036929845809937, + "learning_rate": 0.00011723483530296943, + "loss": 0.20060159683227538, + "mean_token_accuracy": 0.928264719247818, + "num_tokens": 5808279.0, + "step": 2150 + }, + { + "entropy": 0.2522051434218884, + "epoch": 5.392879066912216, + "grad_norm": 0.5887715220451355, + "learning_rate": 0.00011260505696011844, + "loss": 0.19347431182861327, + "mean_token_accuracy": 0.930549119412899, + "num_tokens": 5948774.0, + "step": 2200 + }, + { + "entropy": 0.26358571454882623, + "epoch": 5.515653775322283, + "grad_norm": 0.5937269330024719, + "learning_rate": 0.00010796764243986134, + "loss": 0.20256309509277343, + "mean_token_accuracy": 0.9277606004476547, + "num_tokens": 6081211.0, + "step": 2250 + }, + { + "entropy": 0.2550286700576544, + "epoch": 5.638428483732351, + "grad_norm": 0.6088381409645081, + "learning_rate": 0.00010333107659081343, + "loss": 0.19648813247680663, + "mean_token_accuracy": 0.9293895545601845, + "num_tokens": 6213793.0, + "step": 2300 + }, + { + "entropy": 0.262065271884203, + "epoch": 5.7612031921424185, + "grad_norm": 0.5331980586051941, + "learning_rate": 9.870384270881798e-05, + "loss": 0.2019593048095703, + "mean_token_accuracy": 0.9276418074965477, + "num_tokens": 6350388.0, + "step": 2350 + }, + { + "entropy": 0.2581966924667358, + "epoch": 5.883977900552486, + "grad_norm": 0.6100393533706665, + "learning_rate": 9.409440701547846e-05, + "loss": 0.20058013916015624, + "mean_token_accuracy": 0.928467052578926, + "num_tokens": 6485960.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.31780434923512596, + "eval_loss": 0.6719555258750916, + "eval_mean_token_accuracy": 0.8434631654194423, + "eval_num_tokens": 6610608.0, + "eval_runtime": 108.6811, + "eval_samples_per_second": 12.845, + "eval_steps_per_second": 1.61, + "step": 2448 + }, + { + "entropy": 0.2578806725855406, + "epoch": 6.004910988336403, + "grad_norm": 0.40365952253341675, + "learning_rate": 8.951120316793048e-05, + "loss": 0.19516365051269532, + "mean_token_accuracy": 0.930728108749777, + "num_tokens": 6616117.0, + "step": 2450 + }, + { + "entropy": 0.18210563194006682, + "epoch": 6.12768569674647, + "grad_norm": 0.5063093304634094, + "learning_rate": 8.496261682819517e-05, + "loss": 0.11744135856628418, + "mean_token_accuracy": 0.9580859485268592, + "num_tokens": 6754569.0, + "step": 2500 + }, + { + "entropy": 0.18104823548346757, + "epoch": 6.250460405156538, + "grad_norm": 0.5599386096000671, + "learning_rate": 8.045697032034715e-05, + "loss": 0.11880653381347656, + "mean_token_accuracy": 0.9568264433741569, + "num_tokens": 6893447.0, + "step": 2550 + }, + { + "entropy": 0.18421781908720733, + "epoch": 6.373235113566605, + "grad_norm": 0.5448400378227234, + "learning_rate": 7.600250740356843e-05, + "loss": 0.11898751258850097, + "mean_token_accuracy": 0.9573441597819329, + "num_tokens": 7025939.0, + "step": 2600 + }, + { + "entropy": 0.18598256688565015, + "epoch": 6.496009821976672, + "grad_norm": 0.49465450644493103, + "learning_rate": 7.16073781889491e-05, + "loss": 0.11917055130004883, + "mean_token_accuracy": 0.9567968577146531, + "num_tokens": 7163978.0, + "step": 2650 + }, + { + "entropy": 0.17962845370173455, + "epoch": 6.6187845303867405, + "grad_norm": 0.5606568455696106, + "learning_rate": 6.727962422763097e-05, + "loss": 0.12085281372070313, + "mean_token_accuracy": 0.9565140584111214, + "num_tokens": 7296794.0, + "step": 2700 + }, + { + "entropy": 0.1846459148451686, + "epoch": 6.741559238796808, + "grad_norm": 0.5243878364562988, + "learning_rate": 6.302716379757798e-05, + "loss": 0.12253878593444824, + "mean_token_accuracy": 0.9561242800951004, + "num_tokens": 7429156.0, + "step": 2750 + }, + { + "entropy": 0.17984333034604788, + "epoch": 6.864333947206875, + "grad_norm": 0.5876769423484802, + "learning_rate": 5.885777741589364e-05, + "loss": 0.11955853462219239, + "mean_token_accuracy": 0.9572344574332238, + "num_tokens": 7559658.0, + "step": 2800 + }, + { + "entropy": 0.1771458223089576, + "epoch": 6.987108655616943, + "grad_norm": 0.5043931007385254, + "learning_rate": 5.477909360319267e-05, + "loss": 0.11639300346374512, + "mean_token_accuracy": 0.9580946227908135, + "num_tokens": 7699096.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.2669620673997062, + "eval_loss": 0.774162769317627, + "eval_mean_token_accuracy": 0.8417869962964739, + "eval_num_tokens": 7712376.0, + "eval_runtime": 108.7617, + "eval_samples_per_second": 12.835, + "eval_steps_per_second": 1.609, + "step": 2856 + }, + { + "entropy": 0.15268251057808774, + "epoch": 7.108041743400859, + "grad_norm": 0.421937495470047, + "learning_rate": 5.079857492607303e-05, + "loss": 0.08690091133117676, + "mean_token_accuracy": 0.9695342624853105, + "num_tokens": 7825131.0, + "step": 2900 + }, + { + "entropy": 0.14063113152980805, + "epoch": 7.230816451810927, + "grad_norm": 0.35511091351509094, + "learning_rate": 4.692350434322603e-05, + "loss": 0.07454932689666748, + "mean_token_accuracy": 0.9723883396387101, + "num_tokens": 7963208.0, + "step": 2950 + }, + { + "entropy": 0.13641462391242384, + "epoch": 7.3535911602209945, + "grad_norm": 0.33150219917297363, + "learning_rate": 4.3160971880165837e-05, + "loss": 0.07405156135559082, + "mean_token_accuracy": 0.9731479465961457, + "num_tokens": 8100530.0, + "step": 3000 + }, + { + "entropy": 0.12723070368170739, + "epoch": 7.476365868631062, + "grad_norm": 0.3763302266597748, + "learning_rate": 3.9517861656959645e-05, + "loss": 0.07087192535400391, + "mean_token_accuracy": 0.9746569180488587, + "num_tokens": 8241781.0, + "step": 3050 + }, + { + "entropy": 0.1335552130267024, + "epoch": 7.59914057704113, + "grad_norm": 0.5557640790939331, + "learning_rate": 3.600083929269262e-05, + "loss": 0.07477219581604004, + "mean_token_accuracy": 0.9728479567170143, + "num_tokens": 8374913.0, + "step": 3100 + }, + { + "entropy": 0.13618831865489484, + "epoch": 7.721915285451197, + "grad_norm": 0.508499801158905, + "learning_rate": 3.261633970971359e-05, + "loss": 0.07407661437988282, + "mean_token_accuracy": 0.9726669636368751, + "num_tokens": 8511662.0, + "step": 3150 + }, + { + "entropy": 0.13547602999955416, + "epoch": 7.844689993861264, + "grad_norm": 0.32131460309028625, + "learning_rate": 2.9370555359974904e-05, + "loss": 0.07703304290771484, + "mean_token_accuracy": 0.972478207051754, + "num_tokens": 8645008.0, + "step": 3200 + }, + { + "entropy": 0.1332347623631358, + "epoch": 7.967464702271332, + "grad_norm": 0.4377666413784027, + "learning_rate": 2.6269424895008482e-05, + "loss": 0.07515048027038575, + "mean_token_accuracy": 0.9730277815461159, + "num_tokens": 8778619.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2319684133359364, + "eval_loss": 0.8868282437324524, + "eval_mean_token_accuracy": 0.8414362386294774, + "eval_num_tokens": 8814144.0, + "eval_runtime": 108.7029, + "eval_samples_per_second": 12.842, + "eval_steps_per_second": 1.61, + "step": 3264 + }, + { + "entropy": 0.12603538889948487, + "epoch": 8.088397790055248, + "grad_norm": 0.34813445806503296, + "learning_rate": 2.3318622300268085e-05, + "loss": 0.06378498077392578, + "mean_token_accuracy": 0.9763191284867108, + "num_tokens": 8907693.0, + "step": 3300 + }, + { + "entropy": 0.11884484702721237, + "epoch": 8.211172498465316, + "grad_norm": 0.22056937217712402, + "learning_rate": 2.0523546513718086e-05, + "loss": 0.059115543365478515, + "mean_token_accuracy": 0.9775407165288925, + "num_tokens": 9041538.0, + "step": 3350 + }, + { + "entropy": 0.1213479395210743, + "epoch": 8.333947206875383, + "grad_norm": 0.30439141392707825, + "learning_rate": 1.7889311547662764e-05, + "loss": 0.060338306427001956, + "mean_token_accuracy": 0.9764804869890213, + "num_tokens": 9173381.0, + "step": 3400 + }, + { + "entropy": 0.12034013645723463, + "epoch": 8.456721915285451, + "grad_norm": 0.2019958645105362, + "learning_rate": 1.5420737131890332e-05, + "loss": 0.05998798847198486, + "mean_token_accuracy": 0.9769419169425965, + "num_tokens": 9306859.0, + "step": 3450 + }, + { + "entropy": 0.11213825253769755, + "epoch": 8.579496623695519, + "grad_norm": 0.21801051497459412, + "learning_rate": 1.3122339895251194e-05, + "loss": 0.05649502277374267, + "mean_token_accuracy": 0.9788470858335495, + "num_tokens": 9451826.0, + "step": 3500 + }, + { + "entropy": 0.11963866963982582, + "epoch": 8.702271332105585, + "grad_norm": 0.31590062379837036, + "learning_rate": 1.0998325101804684e-05, + "loss": 0.05895202159881592, + "mean_token_accuracy": 0.9766810277104377, + "num_tokens": 9590049.0, + "step": 3550 + }, + { + "entropy": 0.11513572301715612, + "epoch": 8.825046040515653, + "grad_norm": 0.18179665505886078, + "learning_rate": 9.052578956654935e-06, + "loss": 0.05937095642089844, + "mean_token_accuracy": 0.9773054680228234, + "num_tokens": 9727075.0, + "step": 3600 + }, + { + "entropy": 0.11545982209965587, + "epoch": 8.947820748925722, + "grad_norm": 0.28307580947875977, + "learning_rate": 7.28866149555317e-06, + "loss": 0.0594486665725708, + "mean_token_accuracy": 0.9774915012717247, + "num_tokens": 9862022.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.21108701620783124, + "eval_loss": 0.9910867810249329, + "eval_mean_token_accuracy": 0.8417104813030788, + "eval_num_tokens": 9915912.0, + "eval_runtime": 108.7188, + "eval_samples_per_second": 12.84, + "eval_steps_per_second": 1.61, + "step": 3672 + }, + { + "entropy": 0.11195710187938612, + "epoch": 9.068753836709638, + "grad_norm": 0.21504324674606323, + "learning_rate": 5.709800071275755e-06, + "loss": 0.056673107147216795, + "mean_token_accuracy": 0.9789708807383697, + "num_tokens": 9994836.0, + "step": 3700 + }, + { + "entropy": 0.11649182487279176, + "epoch": 9.191528545119706, + "grad_norm": 0.28711235523223877, + "learning_rate": 4.318883448696035e-06, + "loss": 0.0559059476852417, + "mean_token_accuracy": 0.9785248625278473, + "num_tokens": 10125546.0, + "step": 3750 + }, + { + "entropy": 0.1143949885852635, + "epoch": 9.314303253529772, + "grad_norm": 0.21793009340763092, + "learning_rate": 3.118456519353742e-06, + "loss": 0.05573330402374268, + "mean_token_accuracy": 0.978881264925003, + "num_tokens": 10257820.0, + "step": 3800 + }, + { + "entropy": 0.10931090073660016, + "epoch": 9.43707796193984, + "grad_norm": 0.19231684505939484, + "learning_rate": 2.1107156451925275e-06, + "loss": 0.05488526344299317, + "mean_token_accuracy": 0.9790352180600166, + "num_tokens": 10394276.0, + "step": 3850 + }, + { + "entropy": 0.11902843933552504, + "epoch": 9.559852670349908, + "grad_norm": 0.21660247445106506, + "learning_rate": 1.2975046399847277e-06, + "loss": 0.05692287445068359, + "mean_token_accuracy": 0.9773650997877121, + "num_tokens": 10525083.0, + "step": 3900 + }, + { + "entropy": 0.10933571977540851, + "epoch": 9.682627378759975, + "grad_norm": 0.20566122233867645, + "learning_rate": 6.803113957963157e-07, + "loss": 0.05344192504882812, + "mean_token_accuracy": 0.9791323846578598, + "num_tokens": 10662496.0, + "step": 3950 + }, + { + "entropy": 0.11067215802147984, + "epoch": 9.805402087170043, + "grad_norm": 0.21908625960350037, + "learning_rate": 2.602651606642059e-07, + "loss": 0.05455122470855713, + "mean_token_accuracy": 0.978945572078228, + "num_tokens": 10799808.0, + "step": 4000 + }, + { + "entropy": 0.10899379778653383, + "epoch": 9.92817679558011, + "grad_norm": 0.1999165564775467, + "learning_rate": 3.813447246673462e-08, + "loss": 0.05340607166290283, + "mean_token_accuracy": 0.9794067478179932, + "num_tokens": 10939770.0, + "step": 4050 + }, + { + "epoch": 10.0, + "eval_entropy": 0.20335759120328087, + "eval_loss": 1.0349678993225098, + "eval_mean_token_accuracy": 0.8414907135282244, + "eval_num_tokens": 11017680.0, + "eval_runtime": 108.7896, + "eval_samples_per_second": 12.832, + "eval_steps_per_second": 1.609, + "step": 4080 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.789359670441083e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ad2c3d366bb2a6896132cf4e21f72c1c718d2a42 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.09831666542701797, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7f5aa5dc2a465d5c51cfc9461edf2a1ae0c49e0b --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/trainer_state.json @@ -0,0 +1,216 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 816, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6558652856945992, + "epoch": 0.12277470841006753, + "grad_norm": 0.8294193148612976, + "learning_rate": 2.604481765338724e-05, + "loss": 1.5730807495117187, + "mean_token_accuracy": 0.6558220008015633, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.8554385647177696, + "epoch": 0.24554941682013506, + "grad_norm": 1.029894232749939, + "learning_rate": 5.2621162197659935e-05, + "loss": 0.7961322021484375, + "mean_token_accuracy": 0.7755533090233803, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7380270153284073, + "epoch": 0.3683241252302026, + "grad_norm": 0.6096898913383484, + "learning_rate": 7.919750674193263e-05, + "loss": 0.6843100738525391, + "mean_token_accuracy": 0.7998733684420586, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6932123881578446, + "epoch": 0.4910988336402701, + "grad_norm": 0.5934288501739502, + "learning_rate": 0.00010577385128620532, + "loss": 0.6458084106445312, + "mean_token_accuracy": 0.8083393195271492, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6840551143884659, + "epoch": 0.6138735420503376, + "grad_norm": 0.4665544331073761, + "learning_rate": 0.00013235019583047802, + "loss": 0.6333241653442383, + "mean_token_accuracy": 0.8122684139013291, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6649293206632138, + "epoch": 0.7366482504604052, + "grad_norm": 0.47893014550209045, + "learning_rate": 0.00015892654037475069, + "loss": 0.6107040786743164, + "mean_token_accuracy": 0.8165940269827843, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6492742404341698, + "epoch": 0.8594229588704727, + "grad_norm": 0.6059293746948242, + "learning_rate": 0.0001855028849190234, + "loss": 0.597186050415039, + "mean_token_accuracy": 0.8217281407117843, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.642182088047266, + "epoch": 0.9821976672805403, + "grad_norm": 0.3793066143989563, + "learning_rate": 0.0002120792294632961, + "loss": 0.5949800491333008, + "mean_token_accuracy": 0.8229166463017463, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6541631272860936, + "eval_loss": 0.5903413891792297, + "eval_mean_token_accuracy": 0.8252546640804835, + "eval_num_tokens": 1101768.0, + "eval_runtime": 108.9056, + "eval_samples_per_second": 12.818, + "eval_steps_per_second": 1.607, + "step": 408 + }, + { + "entropy": 0.6077755300829253, + "epoch": 1.1031307550644567, + "grad_norm": 0.5028847455978394, + "learning_rate": 0.00021679626884558217, + "loss": 0.5617346954345703, + "mean_token_accuracy": 0.8301527442665875, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5860125370323658, + "epoch": 1.2259054634745243, + "grad_norm": 0.38494572043418884, + "learning_rate": 0.00021653451093163906, + "loss": 0.5437137985229492, + "mean_token_accuracy": 0.8346565261483192, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5945393888652325, + "epoch": 1.3486801718845918, + "grad_norm": 0.5123576521873474, + "learning_rate": 0.00021607496224450087, + "loss": 0.54191650390625, + "mean_token_accuracy": 0.8335164493322372, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.6083934807777405, + "epoch": 1.4714548802946594, + "grad_norm": 0.46661439538002014, + "learning_rate": 0.000215418463597734, + "loss": 0.5550478744506836, + "mean_token_accuracy": 0.8311341696977615, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5862340961396694, + "epoch": 1.5942295887047269, + "grad_norm": 0.49572765827178955, + "learning_rate": 0.00021456621615453177, + "loss": 0.5297146606445312, + "mean_token_accuracy": 0.8360210624337197, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5832360745966434, + "epoch": 1.7170042971147943, + "grad_norm": 0.35520103573799133, + "learning_rate": 0.0002135197792300053, + "loss": 0.5287040328979492, + "mean_token_accuracy": 0.8368027776479721, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5792283065617084, + "epoch": 1.839779005524862, + "grad_norm": 0.3548352122306824, + "learning_rate": 0.00021228106743818178, + "loss": 0.5251237487792969, + "mean_token_accuracy": 0.8396679371595382, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5694268302619457, + "epoch": 1.9625537139349294, + "grad_norm": 0.3476282060146332, + "learning_rate": 0.00021085234718892933, + "loss": 0.5189918899536132, + "mean_token_accuracy": 0.8398650133609772, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5552395452771868, + "eval_loss": 0.5418588519096375, + "eval_mean_token_accuracy": 0.8379638079234532, + "eval_num_tokens": 2203536.0, + "eval_runtime": 108.6649, + "eval_samples_per_second": 12.847, + "eval_steps_per_second": 1.61, + "step": 816 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.5817695197756416e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbc706f6b7eb2cc5d17ef1de0d7a89ab44c9683 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.08935122001117755, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0ea8ab915d2a3b03cc322fa7783217268343dbf8 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/trainer_state.json @@ -0,0 +1,681 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2856, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4665306943655014, + "epoch": 0.12277470841006753, + "grad_norm": 0.8344842791557312, + "learning_rate": 2.1010394822962346e-05, + "loss": 1.379196014404297, + "mean_token_accuracy": 0.6848452094197274, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.7834547007083893, + "epoch": 0.24554941682013506, + "grad_norm": 1.1248278617858887, + "learning_rate": 4.2449573213740254e-05, + "loss": 0.7345225524902343, + "mean_token_accuracy": 0.7882961884140969, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7194462446868419, + "epoch": 0.3683241252302026, + "grad_norm": 0.6498261094093323, + "learning_rate": 6.388875160451815e-05, + "loss": 0.664264907836914, + "mean_token_accuracy": 0.8042438223958015, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6771699503064156, + "epoch": 0.4910988336402701, + "grad_norm": 0.6653317809104919, + "learning_rate": 8.532792999529606e-05, + "loss": 0.6322076797485352, + "mean_token_accuracy": 0.8130025643110276, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6632554066181183, + "epoch": 0.6138735420503376, + "grad_norm": 0.44293469190597534, + "learning_rate": 0.00010676710838607398, + "loss": 0.617110595703125, + "mean_token_accuracy": 0.8168379080295562, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6508415986597538, + "epoch": 0.7366482504604052, + "grad_norm": 0.49295490980148315, + "learning_rate": 0.00012820628677685187, + "loss": 0.6023779296875, + "mean_token_accuracy": 0.8187075120210647, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6380941693484783, + "epoch": 0.8594229588704727, + "grad_norm": 0.5855488181114197, + "learning_rate": 0.00014964546516762976, + "loss": 0.5912636947631836, + "mean_token_accuracy": 0.8227647486329078, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.6346297286450863, + "epoch": 0.9821976672805403, + "grad_norm": 0.3817698657512665, + "learning_rate": 0.00017108464355840766, + "loss": 0.5902091598510742, + "mean_token_accuracy": 0.8231476756930352, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6600051839011056, + "eval_loss": 0.5857571363449097, + "eval_mean_token_accuracy": 0.8248803649629866, + "eval_num_tokens": 1101768.0, + "eval_runtime": 106.3263, + "eval_samples_per_second": 13.12, + "eval_steps_per_second": 1.646, + "step": 408 + }, + { + "entropy": 0.5965584144374441, + "epoch": 1.1031307550644567, + "grad_norm": 0.5178472399711609, + "learning_rate": 0.00017488988654901867, + "loss": 0.5578182220458985, + "mean_token_accuracy": 0.830260409921559, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5831904768943786, + "epoch": 1.2259054634745243, + "grad_norm": 0.4383259117603302, + "learning_rate": 0.00017467872603358825, + "loss": 0.5399176788330078, + "mean_token_accuracy": 0.8351450267434121, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5885515750944614, + "epoch": 1.3486801718845918, + "grad_norm": 0.5418892502784729, + "learning_rate": 0.00017430800739444695, + "loss": 0.5402648544311524, + "mean_token_accuracy": 0.8344706323742866, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.603163862079382, + "epoch": 1.4714548802946594, + "grad_norm": 0.5235878825187683, + "learning_rate": 0.0001737784089172037, + "loss": 0.5534031677246094, + "mean_token_accuracy": 0.8304652521014213, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5745155589282512, + "epoch": 1.5942295887047269, + "grad_norm": 0.46109989285469055, + "learning_rate": 0.00017309089958207074, + "loss": 0.5251047134399414, + "mean_token_accuracy": 0.8380702176690101, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5756107035279274, + "epoch": 1.7170042971147943, + "grad_norm": 0.38209712505340576, + "learning_rate": 0.00017224673729096835, + "loss": 0.5253059387207031, + "mean_token_accuracy": 0.8379077833890914, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5695902299880982, + "epoch": 1.839779005524862, + "grad_norm": 0.4063270688056946, + "learning_rate": 0.00017124746656600374, + "loss": 0.5199185180664062, + "mean_token_accuracy": 0.8399301299452782, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5635769453644752, + "epoch": 1.9625537139349294, + "grad_norm": 0.4165436029434204, + "learning_rate": 0.00017009491572353503, + "loss": 0.5156089019775391, + "mean_token_accuracy": 0.8404546654224396, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5587481154714312, + "eval_loss": 0.5402795672416687, + "eval_mean_token_accuracy": 0.8386626076698304, + "eval_num_tokens": 2203536.0, + "eval_runtime": 105.9883, + "eval_samples_per_second": 13.162, + "eval_steps_per_second": 1.651, + "step": 816 + }, + { + "entropy": 0.5002549305785126, + "epoch": 2.0834868017188457, + "grad_norm": 0.4061429500579834, + "learning_rate": 0.00016879119352899107, + "loss": 0.4499735641479492, + "mean_token_accuracy": 0.8566478448470837, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.4894501394033432, + "epoch": 2.2062615101289134, + "grad_norm": 0.530322790145874, + "learning_rate": 0.00016733868533856737, + "loss": 0.4376458740234375, + "mean_token_accuracy": 0.859256779551506, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.48188683822751044, + "epoch": 2.329036218538981, + "grad_norm": 0.4338841438293457, + "learning_rate": 0.0001657400487348575, + "loss": 0.435361328125, + "mean_token_accuracy": 0.8610378202795982, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.4992289599776268, + "epoch": 2.4518109269490487, + "grad_norm": 0.44097110629081726, + "learning_rate": 0.00016399820866440517, + "loss": 0.4474017333984375, + "mean_token_accuracy": 0.8565557089447975, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.49334951683878897, + "epoch": 2.574585635359116, + "grad_norm": 0.48084133863449097, + "learning_rate": 0.00016211635208607405, + "loss": 0.440751953125, + "mean_token_accuracy": 0.8580261027812958, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.4898635870218277, + "epoch": 2.6973603437691835, + "grad_norm": 0.40215447545051575, + "learning_rate": 0.00016009792214002617, + "loss": 0.44588775634765626, + "mean_token_accuracy": 0.8572473660111427, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.4932038363814354, + "epoch": 2.820135052179251, + "grad_norm": 0.4465612769126892, + "learning_rate": 0.00015794661184797816, + "loss": 0.4444055938720703, + "mean_token_accuracy": 0.8583759877085686, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.4850301502645016, + "epoch": 2.942909760589319, + "grad_norm": 0.43597331643104553, + "learning_rate": 0.0001556663573562617, + "loss": 0.4382463073730469, + "mean_token_accuracy": 0.8603282216191291, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.4902002043383462, + "eval_loss": 0.5276822447776794, + "eval_mean_token_accuracy": 0.8454030500139509, + "eval_num_tokens": 3305304.0, + "eval_runtime": 106.004, + "eval_samples_per_second": 13.16, + "eval_steps_per_second": 1.651, + "step": 1224 + }, + { + "entropy": 0.4309167733349776, + "epoch": 3.063842848373235, + "grad_norm": 0.4111804664134979, + "learning_rate": 0.00015326133073405068, + "loss": 0.3837854766845703, + "mean_token_accuracy": 0.8741549949355537, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.38693042084574697, + "epoch": 3.1866175567833026, + "grad_norm": 0.46740710735321045, + "learning_rate": 0.00015073593233993186, + "loss": 0.335601806640625, + "mean_token_accuracy": 0.8861287096142769, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.39316988229751587, + "epoch": 3.3093922651933703, + "grad_norm": 0.5308303236961365, + "learning_rate": 0.00014809478277078602, + "loss": 0.34588619232177736, + "mean_token_accuracy": 0.8837816697359085, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.39958367675542833, + "epoch": 3.4321669736034375, + "grad_norm": 0.5955453515052795, + "learning_rate": 0.00014534271440770966, + "loss": 0.34634483337402344, + "mean_token_accuracy": 0.8832473835349083, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.4100498601794243, + "epoch": 3.554941682013505, + "grad_norm": 0.4593118727207184, + "learning_rate": 0.0001424847625744456, + "loss": 0.3560930633544922, + "mean_token_accuracy": 0.8804642978310585, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.39962531872093676, + "epoch": 3.677716390423573, + "grad_norm": 0.4584615230560303, + "learning_rate": 0.00013952615632449963, + "loss": 0.34978450775146486, + "mean_token_accuracy": 0.8825609818100929, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.39326138779520986, + "epoch": 3.80049109883364, + "grad_norm": 0.5227728486061096, + "learning_rate": 0.00013647230887379902, + "loss": 0.3470985794067383, + "mean_token_accuracy": 0.8833131140470505, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.399793985709548, + "epoch": 3.9232658072437077, + "grad_norm": 0.4263511002063751, + "learning_rate": 0.00013332880769639856, + "loss": 0.3550622177124023, + "mean_token_accuracy": 0.8825028780102729, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.41604454040527344, + "eval_loss": 0.5501811504364014, + "eval_mean_token_accuracy": 0.8466314860752651, + "eval_num_tokens": 4407072.0, + "eval_runtime": 105.9531, + "eval_samples_per_second": 13.166, + "eval_steps_per_second": 1.652, + "step": 1632 + }, + { + "entropy": 0.3492097145107192, + "epoch": 4.044198895027624, + "grad_norm": 0.41719651222229004, + "learning_rate": 0.00013010140430135466, + "loss": 0.30388387680053713, + "mean_token_accuracy": 0.8972433387930623, + "num_tokens": 4455549.0, + "step": 1650 + }, + { + "entropy": 0.2980811195075512, + "epoch": 4.166973603437691, + "grad_norm": 0.694450318813324, + "learning_rate": 0.00012679600370947314, + "loss": 0.24677230834960937, + "mean_token_accuracy": 0.9134943392872811, + "num_tokens": 4584218.0, + "step": 1700 + }, + { + "entropy": 0.2911450309306383, + "epoch": 4.2897483118477595, + "grad_norm": 0.4893203675746918, + "learning_rate": 0.00012341865364918372, + "loss": 0.24411561965942383, + "mean_token_accuracy": 0.9145414170622825, + "num_tokens": 4724425.0, + "step": 1750 + }, + { + "entropy": 0.2982170185446739, + "epoch": 4.412523020257827, + "grad_norm": 0.5446121096611023, + "learning_rate": 0.00011997553349130966, + "loss": 0.24687444686889648, + "mean_token_accuracy": 0.9126561579108238, + "num_tokens": 4861552.0, + "step": 1800 + }, + { + "entropy": 0.30126186162233354, + "epoch": 4.535297728667894, + "grad_norm": 0.5465192198753357, + "learning_rate": 0.00011647294294297787, + "loss": 0.2527424621582031, + "mean_token_accuracy": 0.9110122618079185, + "num_tokens": 4993610.0, + "step": 1850 + }, + { + "entropy": 0.2993039993941784, + "epoch": 4.658072437077962, + "grad_norm": 0.6376634836196899, + "learning_rate": 0.00011291729052135527, + "loss": 0.2507955169677734, + "mean_token_accuracy": 0.9124656105041504, + "num_tokens": 5126291.0, + "step": 1900 + }, + { + "entropy": 0.29630616143345834, + "epoch": 4.780847145488029, + "grad_norm": 0.5191586017608643, + "learning_rate": 0.00010931508182830125, + "loss": 0.2503428077697754, + "mean_token_accuracy": 0.9127612486481667, + "num_tokens": 5266076.0, + "step": 1950 + }, + { + "entropy": 0.29707231901586056, + "epoch": 4.903621853898097, + "grad_norm": 0.42329415678977966, + "learning_rate": 0.00010567290764738874, + "loss": 0.24930551528930664, + "mean_token_accuracy": 0.911863605082035, + "num_tokens": 5403761.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.34850709250995093, + "eval_loss": 0.6053183078765869, + "eval_mean_token_accuracy": 0.8461176470347813, + "eval_num_tokens": 5508840.0, + "eval_runtime": 105.9653, + "eval_samples_per_second": 13.165, + "eval_steps_per_second": 1.651, + "step": 2040 + }, + { + "entropy": 0.28518860919523964, + "epoch": 5.024554941682013, + "grad_norm": 0.6316563487052917, + "learning_rate": 0.00010199743188507247, + "loss": 0.23341741561889648, + "mean_token_accuracy": 0.9183166952907736, + "num_tokens": 5535743.0, + "step": 2050 + }, + { + "entropy": 0.20185263987630606, + "epoch": 5.147329650092081, + "grad_norm": 0.6542733311653137, + "learning_rate": 9.829537937806829e-05, + "loss": 0.1496055793762207, + "mean_token_accuracy": 0.9459337556362152, + "num_tokens": 5675294.0, + "step": 2100 + }, + { + "entropy": 0.2101737355440855, + "epoch": 5.270104358502149, + "grad_norm": 0.46600785851478577, + "learning_rate": 9.457352358925075e-05, + "loss": 0.15642745018005372, + "mean_token_accuracy": 0.944338481426239, + "num_tokens": 5808279.0, + "step": 2150 + }, + { + "entropy": 0.200700718536973, + "epoch": 5.392879066912216, + "grad_norm": 0.5562126636505127, + "learning_rate": 9.083867421458257e-05, + "loss": 0.15122636795043945, + "mean_token_accuracy": 0.9447117391228675, + "num_tokens": 5948774.0, + "step": 2200 + }, + { + "entropy": 0.2139152016490698, + "epoch": 5.515653775322283, + "grad_norm": 0.5136730670928955, + "learning_rate": 8.709766472374944e-05, + "loss": 0.16092844009399415, + "mean_token_accuracy": 0.9423242849111557, + "num_tokens": 6081211.0, + "step": 2250 + }, + { + "entropy": 0.20662450045347214, + "epoch": 5.638428483732351, + "grad_norm": 0.6007277965545654, + "learning_rate": 8.335733985729791e-05, + "loss": 0.15495609283447265, + "mean_token_accuracy": 0.9431806969642639, + "num_tokens": 6213793.0, + "step": 2300 + }, + { + "entropy": 0.20562831707298757, + "epoch": 5.7612031921424185, + "grad_norm": 0.5216547846794128, + "learning_rate": 7.962454310315095e-05, + "loss": 0.1568380641937256, + "mean_token_accuracy": 0.9425676354765892, + "num_tokens": 6350388.0, + "step": 2350 + }, + { + "entropy": 0.20440548222512006, + "epoch": 5.883977900552486, + "grad_norm": 0.5724782943725586, + "learning_rate": 7.590610417541581e-05, + "loss": 0.15627790451049806, + "mean_token_accuracy": 0.9437827947735786, + "num_tokens": 6485960.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.2856926244497299, + "eval_loss": 0.7071177959442139, + "eval_mean_token_accuracy": 0.8432521397726876, + "eval_num_tokens": 6610608.0, + "eval_runtime": 105.9542, + "eval_samples_per_second": 13.166, + "eval_steps_per_second": 1.652, + "step": 2448 + }, + { + "entropy": 0.20338259523895186, + "epoch": 6.004910988336403, + "grad_norm": 0.39025962352752686, + "learning_rate": 7.220882651839295e-05, + "loss": 0.15414920806884766, + "mean_token_accuracy": 0.9444228456105073, + "num_tokens": 6616117.0, + "step": 2450 + }, + { + "entropy": 0.13960296589881183, + "epoch": 6.12768569674647, + "grad_norm": 0.4555230438709259, + "learning_rate": 6.85394748586495e-05, + "loss": 0.08990025520324707, + "mean_token_accuracy": 0.968256405889988, + "num_tokens": 6754569.0, + "step": 2500 + }, + { + "entropy": 0.13907674692571162, + "epoch": 6.250460405156538, + "grad_norm": 0.49845439195632935, + "learning_rate": 6.490476282793282e-05, + "loss": 0.08988103866577149, + "mean_token_accuracy": 0.9675386163592339, + "num_tokens": 6893447.0, + "step": 2550 + }, + { + "entropy": 0.143365166708827, + "epoch": 6.373235113566605, + "grad_norm": 0.5541622042655945, + "learning_rate": 6.131134067956953e-05, + "loss": 0.09036359786987305, + "mean_token_accuracy": 0.9669707071781158, + "num_tokens": 7025939.0, + "step": 2600 + }, + { + "entropy": 0.14202470190823077, + "epoch": 6.496009821976672, + "grad_norm": 0.38522830605506897, + "learning_rate": 5.776578312082506e-05, + "loss": 0.08982279777526855, + "mean_token_accuracy": 0.9668073603510856, + "num_tokens": 7163978.0, + "step": 2650 + }, + { + "entropy": 0.14029258273541928, + "epoch": 6.6187845303867405, + "grad_norm": 0.4234164357185364, + "learning_rate": 5.4274577283486146e-05, + "loss": 0.09312637329101563, + "mean_token_accuracy": 0.9669712704420089, + "num_tokens": 7296794.0, + "step": 2700 + }, + { + "entropy": 0.14380223151296379, + "epoch": 6.741559238796808, + "grad_norm": 0.36372771859169006, + "learning_rate": 5.084411085467557e-05, + "loss": 0.09390204429626464, + "mean_token_accuracy": 0.966132538318634, + "num_tokens": 7429156.0, + "step": 2750 + }, + { + "entropy": 0.1392465164512396, + "epoch": 6.864333947206875, + "grad_norm": 0.4069533348083496, + "learning_rate": 4.748066038961626e-05, + "loss": 0.09048093795776367, + "mean_token_accuracy": 0.9678688687086106, + "num_tokens": 7559658.0, + "step": 2800 + }, + { + "entropy": 0.13713553678244353, + "epoch": 6.987108655616943, + "grad_norm": 0.34467506408691406, + "learning_rate": 4.4190379827727676e-05, + "loss": 0.08747495651245117, + "mean_token_accuracy": 0.9685557943582535, + "num_tokens": 7699096.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.22547772577830724, + "eval_loss": 0.8343477249145508, + "eval_mean_token_accuracy": 0.8449994778633118, + "eval_num_tokens": 7712376.0, + "eval_runtime": 105.9168, + "eval_samples_per_second": 13.171, + "eval_steps_per_second": 1.652, + "step": 2856 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.287381281114235e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbc706f6b7eb2cc5d17ef1de0d7a89ab44c9683 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.08935122001117755, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6b1b6582a204829536cc20d79728353c1665d0fd --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/trainer_state.json @@ -0,0 +1,772 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3264, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4665306943655014, + "epoch": 0.12277470841006753, + "grad_norm": 0.8344842791557312, + "learning_rate": 2.1010394822962346e-05, + "loss": 1.379196014404297, + "mean_token_accuracy": 0.6848452094197274, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.7834547007083893, + "epoch": 0.24554941682013506, + "grad_norm": 1.1248278617858887, + "learning_rate": 4.2449573213740254e-05, + "loss": 0.7345225524902343, + "mean_token_accuracy": 0.7882961884140969, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7194462446868419, + "epoch": 0.3683241252302026, + "grad_norm": 0.6498261094093323, + "learning_rate": 6.388875160451815e-05, + "loss": 0.664264907836914, + "mean_token_accuracy": 0.8042438223958015, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6771699503064156, + "epoch": 0.4910988336402701, + "grad_norm": 0.6653317809104919, + "learning_rate": 8.532792999529606e-05, + "loss": 0.6322076797485352, + "mean_token_accuracy": 0.8130025643110276, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6632554066181183, + "epoch": 0.6138735420503376, + "grad_norm": 0.44293469190597534, + "learning_rate": 0.00010676710838607398, + "loss": 0.617110595703125, + "mean_token_accuracy": 0.8168379080295562, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6508415986597538, + "epoch": 0.7366482504604052, + "grad_norm": 0.49295490980148315, + "learning_rate": 0.00012820628677685187, + "loss": 0.6023779296875, + "mean_token_accuracy": 0.8187075120210647, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6380941693484783, + "epoch": 0.8594229588704727, + "grad_norm": 0.5855488181114197, + "learning_rate": 0.00014964546516762976, + "loss": 0.5912636947631836, + "mean_token_accuracy": 0.8227647486329078, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.6346297286450863, + "epoch": 0.9821976672805403, + "grad_norm": 0.3817698657512665, + "learning_rate": 0.00017108464355840766, + "loss": 0.5902091598510742, + "mean_token_accuracy": 0.8231476756930352, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6600051839011056, + "eval_loss": 0.5857571363449097, + "eval_mean_token_accuracy": 0.8248803649629866, + "eval_num_tokens": 1101768.0, + "eval_runtime": 106.3263, + "eval_samples_per_second": 13.12, + "eval_steps_per_second": 1.646, + "step": 408 + }, + { + "entropy": 0.5965584144374441, + "epoch": 1.1031307550644567, + "grad_norm": 0.5178472399711609, + "learning_rate": 0.00017488988654901867, + "loss": 0.5578182220458985, + "mean_token_accuracy": 0.830260409921559, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5831904768943786, + "epoch": 1.2259054634745243, + "grad_norm": 0.4383259117603302, + "learning_rate": 0.00017467872603358825, + "loss": 0.5399176788330078, + "mean_token_accuracy": 0.8351450267434121, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5885515750944614, + "epoch": 1.3486801718845918, + "grad_norm": 0.5418892502784729, + "learning_rate": 0.00017430800739444695, + "loss": 0.5402648544311524, + "mean_token_accuracy": 0.8344706323742866, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.603163862079382, + "epoch": 1.4714548802946594, + "grad_norm": 0.5235878825187683, + "learning_rate": 0.0001737784089172037, + "loss": 0.5534031677246094, + "mean_token_accuracy": 0.8304652521014213, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5745155589282512, + "epoch": 1.5942295887047269, + "grad_norm": 0.46109989285469055, + "learning_rate": 0.00017309089958207074, + "loss": 0.5251047134399414, + "mean_token_accuracy": 0.8380702176690101, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5756107035279274, + "epoch": 1.7170042971147943, + "grad_norm": 0.38209712505340576, + "learning_rate": 0.00017224673729096835, + "loss": 0.5253059387207031, + "mean_token_accuracy": 0.8379077833890914, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5695902299880982, + "epoch": 1.839779005524862, + "grad_norm": 0.4063270688056946, + "learning_rate": 0.00017124746656600374, + "loss": 0.5199185180664062, + "mean_token_accuracy": 0.8399301299452782, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5635769453644752, + "epoch": 1.9625537139349294, + "grad_norm": 0.4165436029434204, + "learning_rate": 0.00017009491572353503, + "loss": 0.5156089019775391, + "mean_token_accuracy": 0.8404546654224396, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5587481154714312, + "eval_loss": 0.5402795672416687, + "eval_mean_token_accuracy": 0.8386626076698304, + "eval_num_tokens": 2203536.0, + "eval_runtime": 105.9883, + "eval_samples_per_second": 13.162, + "eval_steps_per_second": 1.651, + "step": 816 + }, + { + "entropy": 0.5002549305785126, + "epoch": 2.0834868017188457, + "grad_norm": 0.4061429500579834, + "learning_rate": 0.00016879119352899107, + "loss": 0.4499735641479492, + "mean_token_accuracy": 0.8566478448470837, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.4894501394033432, + "epoch": 2.2062615101289134, + "grad_norm": 0.530322790145874, + "learning_rate": 0.00016733868533856737, + "loss": 0.4376458740234375, + "mean_token_accuracy": 0.859256779551506, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.48188683822751044, + "epoch": 2.329036218538981, + "grad_norm": 0.4338841438293457, + "learning_rate": 0.0001657400487348575, + "loss": 0.435361328125, + "mean_token_accuracy": 0.8610378202795982, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.4992289599776268, + "epoch": 2.4518109269490487, + "grad_norm": 0.44097110629081726, + "learning_rate": 0.00016399820866440517, + "loss": 0.4474017333984375, + "mean_token_accuracy": 0.8565557089447975, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.49334951683878897, + "epoch": 2.574585635359116, + "grad_norm": 0.48084133863449097, + "learning_rate": 0.00016211635208607405, + "loss": 0.440751953125, + "mean_token_accuracy": 0.8580261027812958, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.4898635870218277, + "epoch": 2.6973603437691835, + "grad_norm": 0.40215447545051575, + "learning_rate": 0.00016009792214002617, + "loss": 0.44588775634765626, + "mean_token_accuracy": 0.8572473660111427, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.4932038363814354, + "epoch": 2.820135052179251, + "grad_norm": 0.4465612769126892, + "learning_rate": 0.00015794661184797816, + "loss": 0.4444055938720703, + "mean_token_accuracy": 0.8583759877085686, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.4850301502645016, + "epoch": 2.942909760589319, + "grad_norm": 0.43597331643104553, + "learning_rate": 0.0001556663573562617, + "loss": 0.4382463073730469, + "mean_token_accuracy": 0.8603282216191291, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.4902002043383462, + "eval_loss": 0.5276822447776794, + "eval_mean_token_accuracy": 0.8454030500139509, + "eval_num_tokens": 3305304.0, + "eval_runtime": 106.004, + "eval_samples_per_second": 13.16, + "eval_steps_per_second": 1.651, + "step": 1224 + }, + { + "entropy": 0.4309167733349776, + "epoch": 3.063842848373235, + "grad_norm": 0.4111804664134979, + "learning_rate": 0.00015326133073405068, + "loss": 0.3837854766845703, + "mean_token_accuracy": 0.8741549949355537, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.38693042084574697, + "epoch": 3.1866175567833026, + "grad_norm": 0.46740710735321045, + "learning_rate": 0.00015073593233993186, + "loss": 0.335601806640625, + "mean_token_accuracy": 0.8861287096142769, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.39316988229751587, + "epoch": 3.3093922651933703, + "grad_norm": 0.5308303236961365, + "learning_rate": 0.00014809478277078602, + "loss": 0.34588619232177736, + "mean_token_accuracy": 0.8837816697359085, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.39958367675542833, + "epoch": 3.4321669736034375, + "grad_norm": 0.5955453515052795, + "learning_rate": 0.00014534271440770966, + "loss": 0.34634483337402344, + "mean_token_accuracy": 0.8832473835349083, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.4100498601794243, + "epoch": 3.554941682013505, + "grad_norm": 0.4593118727207184, + "learning_rate": 0.0001424847625744456, + "loss": 0.3560930633544922, + "mean_token_accuracy": 0.8804642978310585, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.39962531872093676, + "epoch": 3.677716390423573, + "grad_norm": 0.4584615230560303, + "learning_rate": 0.00013952615632449963, + "loss": 0.34978450775146486, + "mean_token_accuracy": 0.8825609818100929, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.39326138779520986, + "epoch": 3.80049109883364, + "grad_norm": 0.5227728486061096, + "learning_rate": 0.00013647230887379902, + "loss": 0.3470985794067383, + "mean_token_accuracy": 0.8833131140470505, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.399793985709548, + "epoch": 3.9232658072437077, + "grad_norm": 0.4263511002063751, + "learning_rate": 0.00013332880769639856, + "loss": 0.3550622177124023, + "mean_token_accuracy": 0.8825028780102729, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.41604454040527344, + "eval_loss": 0.5501811504364014, + "eval_mean_token_accuracy": 0.8466314860752651, + "eval_num_tokens": 4407072.0, + "eval_runtime": 105.9531, + "eval_samples_per_second": 13.166, + "eval_steps_per_second": 1.652, + "step": 1632 + }, + { + "entropy": 0.3492097145107192, + "epoch": 4.044198895027624, + "grad_norm": 0.41719651222229004, + "learning_rate": 0.00013010140430135466, + "loss": 0.30388387680053713, + "mean_token_accuracy": 0.8972433387930623, + "num_tokens": 4455549.0, + "step": 1650 + }, + { + "entropy": 0.2980811195075512, + "epoch": 4.166973603437691, + "grad_norm": 0.694450318813324, + "learning_rate": 0.00012679600370947314, + "loss": 0.24677230834960937, + "mean_token_accuracy": 0.9134943392872811, + "num_tokens": 4584218.0, + "step": 1700 + }, + { + "entropy": 0.2911450309306383, + "epoch": 4.2897483118477595, + "grad_norm": 0.4893203675746918, + "learning_rate": 0.00012341865364918372, + "loss": 0.24411561965942383, + "mean_token_accuracy": 0.9145414170622825, + "num_tokens": 4724425.0, + "step": 1750 + }, + { + "entropy": 0.2982170185446739, + "epoch": 4.412523020257827, + "grad_norm": 0.5446121096611023, + "learning_rate": 0.00011997553349130966, + "loss": 0.24687444686889648, + "mean_token_accuracy": 0.9126561579108238, + "num_tokens": 4861552.0, + "step": 1800 + }, + { + "entropy": 0.30126186162233354, + "epoch": 4.535297728667894, + "grad_norm": 0.5465192198753357, + "learning_rate": 0.00011647294294297787, + "loss": 0.2527424621582031, + "mean_token_accuracy": 0.9110122618079185, + "num_tokens": 4993610.0, + "step": 1850 + }, + { + "entropy": 0.2993039993941784, + "epoch": 4.658072437077962, + "grad_norm": 0.6376634836196899, + "learning_rate": 0.00011291729052135527, + "loss": 0.2507955169677734, + "mean_token_accuracy": 0.9124656105041504, + "num_tokens": 5126291.0, + "step": 1900 + }, + { + "entropy": 0.29630616143345834, + "epoch": 4.780847145488029, + "grad_norm": 0.5191586017608643, + "learning_rate": 0.00010931508182830125, + "loss": 0.2503428077697754, + "mean_token_accuracy": 0.9127612486481667, + "num_tokens": 5266076.0, + "step": 1950 + }, + { + "entropy": 0.29707231901586056, + "epoch": 4.903621853898097, + "grad_norm": 0.42329415678977966, + "learning_rate": 0.00010567290764738874, + "loss": 0.24930551528930664, + "mean_token_accuracy": 0.911863605082035, + "num_tokens": 5403761.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.34850709250995093, + "eval_loss": 0.6053183078765869, + "eval_mean_token_accuracy": 0.8461176470347813, + "eval_num_tokens": 5508840.0, + "eval_runtime": 105.9653, + "eval_samples_per_second": 13.165, + "eval_steps_per_second": 1.651, + "step": 2040 + }, + { + "entropy": 0.28518860919523964, + "epoch": 5.024554941682013, + "grad_norm": 0.6316563487052917, + "learning_rate": 0.00010199743188507247, + "loss": 0.23341741561889648, + "mean_token_accuracy": 0.9183166952907736, + "num_tokens": 5535743.0, + "step": 2050 + }, + { + "entropy": 0.20185263987630606, + "epoch": 5.147329650092081, + "grad_norm": 0.6542733311653137, + "learning_rate": 9.829537937806829e-05, + "loss": 0.1496055793762207, + "mean_token_accuracy": 0.9459337556362152, + "num_tokens": 5675294.0, + "step": 2100 + }, + { + "entropy": 0.2101737355440855, + "epoch": 5.270104358502149, + "grad_norm": 0.46600785851478577, + "learning_rate": 9.457352358925075e-05, + "loss": 0.15642745018005372, + "mean_token_accuracy": 0.944338481426239, + "num_tokens": 5808279.0, + "step": 2150 + }, + { + "entropy": 0.200700718536973, + "epoch": 5.392879066912216, + "grad_norm": 0.5562126636505127, + "learning_rate": 9.083867421458257e-05, + "loss": 0.15122636795043945, + "mean_token_accuracy": 0.9447117391228675, + "num_tokens": 5948774.0, + "step": 2200 + }, + { + "entropy": 0.2139152016490698, + "epoch": 5.515653775322283, + "grad_norm": 0.5136730670928955, + "learning_rate": 8.709766472374944e-05, + "loss": 0.16092844009399415, + "mean_token_accuracy": 0.9423242849111557, + "num_tokens": 6081211.0, + "step": 2250 + }, + { + "entropy": 0.20662450045347214, + "epoch": 5.638428483732351, + "grad_norm": 0.6007277965545654, + "learning_rate": 8.335733985729791e-05, + "loss": 0.15495609283447265, + "mean_token_accuracy": 0.9431806969642639, + "num_tokens": 6213793.0, + "step": 2300 + }, + { + "entropy": 0.20562831707298757, + "epoch": 5.7612031921424185, + "grad_norm": 0.5216547846794128, + "learning_rate": 7.962454310315095e-05, + "loss": 0.1568380641937256, + "mean_token_accuracy": 0.9425676354765892, + "num_tokens": 6350388.0, + "step": 2350 + }, + { + "entropy": 0.20440548222512006, + "epoch": 5.883977900552486, + "grad_norm": 0.5724782943725586, + "learning_rate": 7.590610417541581e-05, + "loss": 0.15627790451049806, + "mean_token_accuracy": 0.9437827947735786, + "num_tokens": 6485960.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.2856926244497299, + "eval_loss": 0.7071177959442139, + "eval_mean_token_accuracy": 0.8432521397726876, + "eval_num_tokens": 6610608.0, + "eval_runtime": 105.9542, + "eval_samples_per_second": 13.166, + "eval_steps_per_second": 1.652, + "step": 2448 + }, + { + "entropy": 0.20338259523895186, + "epoch": 6.004910988336403, + "grad_norm": 0.39025962352752686, + "learning_rate": 7.220882651839295e-05, + "loss": 0.15414920806884766, + "mean_token_accuracy": 0.9444228456105073, + "num_tokens": 6616117.0, + "step": 2450 + }, + { + "entropy": 0.13960296589881183, + "epoch": 6.12768569674647, + "grad_norm": 0.4555230438709259, + "learning_rate": 6.85394748586495e-05, + "loss": 0.08990025520324707, + "mean_token_accuracy": 0.968256405889988, + "num_tokens": 6754569.0, + "step": 2500 + }, + { + "entropy": 0.13907674692571162, + "epoch": 6.250460405156538, + "grad_norm": 0.49845439195632935, + "learning_rate": 6.490476282793282e-05, + "loss": 0.08988103866577149, + "mean_token_accuracy": 0.9675386163592339, + "num_tokens": 6893447.0, + "step": 2550 + }, + { + "entropy": 0.143365166708827, + "epoch": 6.373235113566605, + "grad_norm": 0.5541622042655945, + "learning_rate": 6.131134067956953e-05, + "loss": 0.09036359786987305, + "mean_token_accuracy": 0.9669707071781158, + "num_tokens": 7025939.0, + "step": 2600 + }, + { + "entropy": 0.14202470190823077, + "epoch": 6.496009821976672, + "grad_norm": 0.38522830605506897, + "learning_rate": 5.776578312082506e-05, + "loss": 0.08982279777526855, + "mean_token_accuracy": 0.9668073603510856, + "num_tokens": 7163978.0, + "step": 2650 + }, + { + "entropy": 0.14029258273541928, + "epoch": 6.6187845303867405, + "grad_norm": 0.4234164357185364, + "learning_rate": 5.4274577283486146e-05, + "loss": 0.09312637329101563, + "mean_token_accuracy": 0.9669712704420089, + "num_tokens": 7296794.0, + "step": 2700 + }, + { + "entropy": 0.14380223151296379, + "epoch": 6.741559238796808, + "grad_norm": 0.36372771859169006, + "learning_rate": 5.084411085467557e-05, + "loss": 0.09390204429626464, + "mean_token_accuracy": 0.966132538318634, + "num_tokens": 7429156.0, + "step": 2750 + }, + { + "entropy": 0.1392465164512396, + "epoch": 6.864333947206875, + "grad_norm": 0.4069533348083496, + "learning_rate": 4.748066038961626e-05, + "loss": 0.09048093795776367, + "mean_token_accuracy": 0.9678688687086106, + "num_tokens": 7559658.0, + "step": 2800 + }, + { + "entropy": 0.13713553678244353, + "epoch": 6.987108655616943, + "grad_norm": 0.34467506408691406, + "learning_rate": 4.4190379827727676e-05, + "loss": 0.08747495651245117, + "mean_token_accuracy": 0.9685557943582535, + "num_tokens": 7699096.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.22547772577830724, + "eval_loss": 0.8343477249145508, + "eval_mean_token_accuracy": 0.8449994778633118, + "eval_num_tokens": 7712376.0, + "eval_runtime": 105.9168, + "eval_samples_per_second": 13.171, + "eval_steps_per_second": 1.652, + "step": 2856 + }, + { + "entropy": 0.12294510751962662, + "epoch": 7.108041743400859, + "grad_norm": 0.23788245022296906, + "learning_rate": 4.097928923306641e-05, + "loss": 0.07265013217926025, + "mean_token_accuracy": 0.9740867614746094, + "num_tokens": 7825131.0, + "step": 2900 + }, + { + "entropy": 0.11531499886885285, + "epoch": 7.230816451810927, + "grad_norm": 0.23746679723262787, + "learning_rate": 3.785326377971202e-05, + "loss": 0.0625599193572998, + "mean_token_accuracy": 0.9764253079891205, + "num_tokens": 7963208.0, + "step": 2950 + }, + { + "entropy": 0.111452647857368, + "epoch": 7.3535911602209945, + "grad_norm": 0.23431439697742462, + "learning_rate": 3.4818023002250615e-05, + "loss": 0.0625885009765625, + "mean_token_accuracy": 0.9765455546975136, + "num_tokens": 8100530.0, + "step": 3000 + }, + { + "entropy": 0.10402006102725864, + "epoch": 7.476365868631062, + "grad_norm": 0.29770365357398987, + "learning_rate": 3.187912033102466e-05, + "loss": 0.05990049839019775, + "mean_token_accuracy": 0.9781030318140984, + "num_tokens": 8241781.0, + "step": 3050 + }, + { + "entropy": 0.11040891058743, + "epoch": 7.59914057704113, + "grad_norm": 0.32297083735466003, + "learning_rate": 2.9041932931295312e-05, + "loss": 0.06435544013977051, + "mean_token_accuracy": 0.9758345261216164, + "num_tokens": 8374913.0, + "step": 3100 + }, + { + "entropy": 0.11264785576611758, + "epoch": 7.721915285451197, + "grad_norm": 0.37468084692955017, + "learning_rate": 2.6311651864908477e-05, + "loss": 0.06344935417175293, + "mean_token_accuracy": 0.9757431983947754, + "num_tokens": 8511662.0, + "step": 3150 + }, + { + "entropy": 0.11280227556824685, + "epoch": 7.844689993861264, + "grad_norm": 0.2595016658306122, + "learning_rate": 2.3693272592464892e-05, + "loss": 0.06645034790039063, + "mean_token_accuracy": 0.9757053011655807, + "num_tokens": 8645008.0, + "step": 3200 + }, + { + "entropy": 0.1106757521815598, + "epoch": 7.967464702271332, + "grad_norm": 0.3099309206008911, + "learning_rate": 2.1191585833372245e-05, + "loss": 0.06522040367126465, + "mean_token_accuracy": 0.9758621609210968, + "num_tokens": 8778619.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.20031422572476523, + "eval_loss": 0.9309563040733337, + "eval_mean_token_accuracy": 0.8455711133139474, + "eval_num_tokens": 8814144.0, + "eval_runtime": 105.9709, + "eval_samples_per_second": 13.164, + "eval_steps_per_second": 1.651, + "step": 3264 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.4709803977030042e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbc706f6b7eb2cc5d17ef1de0d7a89ab44c9683 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.08935122001117755, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fdd3dd183d26ee863d315d95c87e53a145aa7653 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/trainer_state.json @@ -0,0 +1,863 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3672, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4665306943655014, + "epoch": 0.12277470841006753, + "grad_norm": 0.8344842791557312, + "learning_rate": 2.1010394822962346e-05, + "loss": 1.379196014404297, + "mean_token_accuracy": 0.6848452094197274, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.7834547007083893, + "epoch": 0.24554941682013506, + "grad_norm": 1.1248278617858887, + "learning_rate": 4.2449573213740254e-05, + "loss": 0.7345225524902343, + "mean_token_accuracy": 0.7882961884140969, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7194462446868419, + "epoch": 0.3683241252302026, + "grad_norm": 0.6498261094093323, + "learning_rate": 6.388875160451815e-05, + "loss": 0.664264907836914, + "mean_token_accuracy": 0.8042438223958015, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6771699503064156, + "epoch": 0.4910988336402701, + "grad_norm": 0.6653317809104919, + "learning_rate": 8.532792999529606e-05, + "loss": 0.6322076797485352, + "mean_token_accuracy": 0.8130025643110276, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6632554066181183, + "epoch": 0.6138735420503376, + "grad_norm": 0.44293469190597534, + "learning_rate": 0.00010676710838607398, + "loss": 0.617110595703125, + "mean_token_accuracy": 0.8168379080295562, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6508415986597538, + "epoch": 0.7366482504604052, + "grad_norm": 0.49295490980148315, + "learning_rate": 0.00012820628677685187, + "loss": 0.6023779296875, + "mean_token_accuracy": 0.8187075120210647, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6380941693484783, + "epoch": 0.8594229588704727, + "grad_norm": 0.5855488181114197, + "learning_rate": 0.00014964546516762976, + "loss": 0.5912636947631836, + "mean_token_accuracy": 0.8227647486329078, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.6346297286450863, + "epoch": 0.9821976672805403, + "grad_norm": 0.3817698657512665, + "learning_rate": 0.00017108464355840766, + "loss": 0.5902091598510742, + "mean_token_accuracy": 0.8231476756930352, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6600051839011056, + "eval_loss": 0.5857571363449097, + "eval_mean_token_accuracy": 0.8248803649629866, + "eval_num_tokens": 1101768.0, + "eval_runtime": 106.3263, + "eval_samples_per_second": 13.12, + "eval_steps_per_second": 1.646, + "step": 408 + }, + { + "entropy": 0.5965584144374441, + "epoch": 1.1031307550644567, + "grad_norm": 0.5178472399711609, + "learning_rate": 0.00017488988654901867, + "loss": 0.5578182220458985, + "mean_token_accuracy": 0.830260409921559, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5831904768943786, + "epoch": 1.2259054634745243, + "grad_norm": 0.4383259117603302, + "learning_rate": 0.00017467872603358825, + "loss": 0.5399176788330078, + "mean_token_accuracy": 0.8351450267434121, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5885515750944614, + "epoch": 1.3486801718845918, + "grad_norm": 0.5418892502784729, + "learning_rate": 0.00017430800739444695, + "loss": 0.5402648544311524, + "mean_token_accuracy": 0.8344706323742866, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.603163862079382, + "epoch": 1.4714548802946594, + "grad_norm": 0.5235878825187683, + "learning_rate": 0.0001737784089172037, + "loss": 0.5534031677246094, + "mean_token_accuracy": 0.8304652521014213, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5745155589282512, + "epoch": 1.5942295887047269, + "grad_norm": 0.46109989285469055, + "learning_rate": 0.00017309089958207074, + "loss": 0.5251047134399414, + "mean_token_accuracy": 0.8380702176690101, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5756107035279274, + "epoch": 1.7170042971147943, + "grad_norm": 0.38209712505340576, + "learning_rate": 0.00017224673729096835, + "loss": 0.5253059387207031, + "mean_token_accuracy": 0.8379077833890914, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5695902299880982, + "epoch": 1.839779005524862, + "grad_norm": 0.4063270688056946, + "learning_rate": 0.00017124746656600374, + "loss": 0.5199185180664062, + "mean_token_accuracy": 0.8399301299452782, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5635769453644752, + "epoch": 1.9625537139349294, + "grad_norm": 0.4165436029434204, + "learning_rate": 0.00017009491572353503, + "loss": 0.5156089019775391, + "mean_token_accuracy": 0.8404546654224396, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5587481154714312, + "eval_loss": 0.5402795672416687, + "eval_mean_token_accuracy": 0.8386626076698304, + "eval_num_tokens": 2203536.0, + "eval_runtime": 105.9883, + "eval_samples_per_second": 13.162, + "eval_steps_per_second": 1.651, + "step": 816 + }, + { + "entropy": 0.5002549305785126, + "epoch": 2.0834868017188457, + "grad_norm": 0.4061429500579834, + "learning_rate": 0.00016879119352899107, + "loss": 0.4499735641479492, + "mean_token_accuracy": 0.8566478448470837, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.4894501394033432, + "epoch": 2.2062615101289134, + "grad_norm": 0.530322790145874, + "learning_rate": 0.00016733868533856737, + "loss": 0.4376458740234375, + "mean_token_accuracy": 0.859256779551506, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.48188683822751044, + "epoch": 2.329036218538981, + "grad_norm": 0.4338841438293457, + "learning_rate": 0.0001657400487348575, + "loss": 0.435361328125, + "mean_token_accuracy": 0.8610378202795982, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.4992289599776268, + "epoch": 2.4518109269490487, + "grad_norm": 0.44097110629081726, + "learning_rate": 0.00016399820866440517, + "loss": 0.4474017333984375, + "mean_token_accuracy": 0.8565557089447975, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.49334951683878897, + "epoch": 2.574585635359116, + "grad_norm": 0.48084133863449097, + "learning_rate": 0.00016211635208607405, + "loss": 0.440751953125, + "mean_token_accuracy": 0.8580261027812958, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.4898635870218277, + "epoch": 2.6973603437691835, + "grad_norm": 0.40215447545051575, + "learning_rate": 0.00016009792214002617, + "loss": 0.44588775634765626, + "mean_token_accuracy": 0.8572473660111427, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.4932038363814354, + "epoch": 2.820135052179251, + "grad_norm": 0.4465612769126892, + "learning_rate": 0.00015794661184797816, + "loss": 0.4444055938720703, + "mean_token_accuracy": 0.8583759877085686, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.4850301502645016, + "epoch": 2.942909760589319, + "grad_norm": 0.43597331643104553, + "learning_rate": 0.0001556663573562617, + "loss": 0.4382463073730469, + "mean_token_accuracy": 0.8603282216191291, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.4902002043383462, + "eval_loss": 0.5276822447776794, + "eval_mean_token_accuracy": 0.8454030500139509, + "eval_num_tokens": 3305304.0, + "eval_runtime": 106.004, + "eval_samples_per_second": 13.16, + "eval_steps_per_second": 1.651, + "step": 1224 + }, + { + "entropy": 0.4309167733349776, + "epoch": 3.063842848373235, + "grad_norm": 0.4111804664134979, + "learning_rate": 0.00015326133073405068, + "loss": 0.3837854766845703, + "mean_token_accuracy": 0.8741549949355537, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.38693042084574697, + "epoch": 3.1866175567833026, + "grad_norm": 0.46740710735321045, + "learning_rate": 0.00015073593233993186, + "loss": 0.335601806640625, + "mean_token_accuracy": 0.8861287096142769, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.39316988229751587, + "epoch": 3.3093922651933703, + "grad_norm": 0.5308303236961365, + "learning_rate": 0.00014809478277078602, + "loss": 0.34588619232177736, + "mean_token_accuracy": 0.8837816697359085, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.39958367675542833, + "epoch": 3.4321669736034375, + "grad_norm": 0.5955453515052795, + "learning_rate": 0.00014534271440770966, + "loss": 0.34634483337402344, + "mean_token_accuracy": 0.8832473835349083, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.4100498601794243, + "epoch": 3.554941682013505, + "grad_norm": 0.4593118727207184, + "learning_rate": 0.0001424847625744456, + "loss": 0.3560930633544922, + "mean_token_accuracy": 0.8804642978310585, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.39962531872093676, + "epoch": 3.677716390423573, + "grad_norm": 0.4584615230560303, + "learning_rate": 0.00013952615632449963, + "loss": 0.34978450775146486, + "mean_token_accuracy": 0.8825609818100929, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.39326138779520986, + "epoch": 3.80049109883364, + "grad_norm": 0.5227728486061096, + "learning_rate": 0.00013647230887379902, + "loss": 0.3470985794067383, + "mean_token_accuracy": 0.8833131140470505, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.399793985709548, + "epoch": 3.9232658072437077, + "grad_norm": 0.4263511002063751, + "learning_rate": 0.00013332880769639856, + "loss": 0.3550622177124023, + "mean_token_accuracy": 0.8825028780102729, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.41604454040527344, + "eval_loss": 0.5501811504364014, + "eval_mean_token_accuracy": 0.8466314860752651, + "eval_num_tokens": 4407072.0, + "eval_runtime": 105.9531, + "eval_samples_per_second": 13.166, + "eval_steps_per_second": 1.652, + "step": 1632 + }, + { + "entropy": 0.3492097145107192, + "epoch": 4.044198895027624, + "grad_norm": 0.41719651222229004, + "learning_rate": 0.00013010140430135466, + "loss": 0.30388387680053713, + "mean_token_accuracy": 0.8972433387930623, + "num_tokens": 4455549.0, + "step": 1650 + }, + { + "entropy": 0.2980811195075512, + "epoch": 4.166973603437691, + "grad_norm": 0.694450318813324, + "learning_rate": 0.00012679600370947314, + "loss": 0.24677230834960937, + "mean_token_accuracy": 0.9134943392872811, + "num_tokens": 4584218.0, + "step": 1700 + }, + { + "entropy": 0.2911450309306383, + "epoch": 4.2897483118477595, + "grad_norm": 0.4893203675746918, + "learning_rate": 0.00012341865364918372, + "loss": 0.24411561965942383, + "mean_token_accuracy": 0.9145414170622825, + "num_tokens": 4724425.0, + "step": 1750 + }, + { + "entropy": 0.2982170185446739, + "epoch": 4.412523020257827, + "grad_norm": 0.5446121096611023, + "learning_rate": 0.00011997553349130966, + "loss": 0.24687444686889648, + "mean_token_accuracy": 0.9126561579108238, + "num_tokens": 4861552.0, + "step": 1800 + }, + { + "entropy": 0.30126186162233354, + "epoch": 4.535297728667894, + "grad_norm": 0.5465192198753357, + "learning_rate": 0.00011647294294297787, + "loss": 0.2527424621582031, + "mean_token_accuracy": 0.9110122618079185, + "num_tokens": 4993610.0, + "step": 1850 + }, + { + "entropy": 0.2993039993941784, + "epoch": 4.658072437077962, + "grad_norm": 0.6376634836196899, + "learning_rate": 0.00011291729052135527, + "loss": 0.2507955169677734, + "mean_token_accuracy": 0.9124656105041504, + "num_tokens": 5126291.0, + "step": 1900 + }, + { + "entropy": 0.29630616143345834, + "epoch": 4.780847145488029, + "grad_norm": 0.5191586017608643, + "learning_rate": 0.00010931508182830125, + "loss": 0.2503428077697754, + "mean_token_accuracy": 0.9127612486481667, + "num_tokens": 5266076.0, + "step": 1950 + }, + { + "entropy": 0.29707231901586056, + "epoch": 4.903621853898097, + "grad_norm": 0.42329415678977966, + "learning_rate": 0.00010567290764738874, + "loss": 0.24930551528930664, + "mean_token_accuracy": 0.911863605082035, + "num_tokens": 5403761.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.34850709250995093, + "eval_loss": 0.6053183078765869, + "eval_mean_token_accuracy": 0.8461176470347813, + "eval_num_tokens": 5508840.0, + "eval_runtime": 105.9653, + "eval_samples_per_second": 13.165, + "eval_steps_per_second": 1.651, + "step": 2040 + }, + { + "entropy": 0.28518860919523964, + "epoch": 5.024554941682013, + "grad_norm": 0.6316563487052917, + "learning_rate": 0.00010199743188507247, + "loss": 0.23341741561889648, + "mean_token_accuracy": 0.9183166952907736, + "num_tokens": 5535743.0, + "step": 2050 + }, + { + "entropy": 0.20185263987630606, + "epoch": 5.147329650092081, + "grad_norm": 0.6542733311653137, + "learning_rate": 9.829537937806829e-05, + "loss": 0.1496055793762207, + "mean_token_accuracy": 0.9459337556362152, + "num_tokens": 5675294.0, + "step": 2100 + }, + { + "entropy": 0.2101737355440855, + "epoch": 5.270104358502149, + "grad_norm": 0.46600785851478577, + "learning_rate": 9.457352358925075e-05, + "loss": 0.15642745018005372, + "mean_token_accuracy": 0.944338481426239, + "num_tokens": 5808279.0, + "step": 2150 + }, + { + "entropy": 0.200700718536973, + "epoch": 5.392879066912216, + "grad_norm": 0.5562126636505127, + "learning_rate": 9.083867421458257e-05, + "loss": 0.15122636795043945, + "mean_token_accuracy": 0.9447117391228675, + "num_tokens": 5948774.0, + "step": 2200 + }, + { + "entropy": 0.2139152016490698, + "epoch": 5.515653775322283, + "grad_norm": 0.5136730670928955, + "learning_rate": 8.709766472374944e-05, + "loss": 0.16092844009399415, + "mean_token_accuracy": 0.9423242849111557, + "num_tokens": 6081211.0, + "step": 2250 + }, + { + "entropy": 0.20662450045347214, + "epoch": 5.638428483732351, + "grad_norm": 0.6007277965545654, + "learning_rate": 8.335733985729791e-05, + "loss": 0.15495609283447265, + "mean_token_accuracy": 0.9431806969642639, + "num_tokens": 6213793.0, + "step": 2300 + }, + { + "entropy": 0.20562831707298757, + "epoch": 5.7612031921424185, + "grad_norm": 0.5216547846794128, + "learning_rate": 7.962454310315095e-05, + "loss": 0.1568380641937256, + "mean_token_accuracy": 0.9425676354765892, + "num_tokens": 6350388.0, + "step": 2350 + }, + { + "entropy": 0.20440548222512006, + "epoch": 5.883977900552486, + "grad_norm": 0.5724782943725586, + "learning_rate": 7.590610417541581e-05, + "loss": 0.15627790451049806, + "mean_token_accuracy": 0.9437827947735786, + "num_tokens": 6485960.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.2856926244497299, + "eval_loss": 0.7071177959442139, + "eval_mean_token_accuracy": 0.8432521397726876, + "eval_num_tokens": 6610608.0, + "eval_runtime": 105.9542, + "eval_samples_per_second": 13.166, + "eval_steps_per_second": 1.652, + "step": 2448 + }, + { + "entropy": 0.20338259523895186, + "epoch": 6.004910988336403, + "grad_norm": 0.39025962352752686, + "learning_rate": 7.220882651839295e-05, + "loss": 0.15414920806884766, + "mean_token_accuracy": 0.9444228456105073, + "num_tokens": 6616117.0, + "step": 2450 + }, + { + "entropy": 0.13960296589881183, + "epoch": 6.12768569674647, + "grad_norm": 0.4555230438709259, + "learning_rate": 6.85394748586495e-05, + "loss": 0.08990025520324707, + "mean_token_accuracy": 0.968256405889988, + "num_tokens": 6754569.0, + "step": 2500 + }, + { + "entropy": 0.13907674692571162, + "epoch": 6.250460405156538, + "grad_norm": 0.49845439195632935, + "learning_rate": 6.490476282793282e-05, + "loss": 0.08988103866577149, + "mean_token_accuracy": 0.9675386163592339, + "num_tokens": 6893447.0, + "step": 2550 + }, + { + "entropy": 0.143365166708827, + "epoch": 6.373235113566605, + "grad_norm": 0.5541622042655945, + "learning_rate": 6.131134067956953e-05, + "loss": 0.09036359786987305, + "mean_token_accuracy": 0.9669707071781158, + "num_tokens": 7025939.0, + "step": 2600 + }, + { + "entropy": 0.14202470190823077, + "epoch": 6.496009821976672, + "grad_norm": 0.38522830605506897, + "learning_rate": 5.776578312082506e-05, + "loss": 0.08982279777526855, + "mean_token_accuracy": 0.9668073603510856, + "num_tokens": 7163978.0, + "step": 2650 + }, + { + "entropy": 0.14029258273541928, + "epoch": 6.6187845303867405, + "grad_norm": 0.4234164357185364, + "learning_rate": 5.4274577283486146e-05, + "loss": 0.09312637329101563, + "mean_token_accuracy": 0.9669712704420089, + "num_tokens": 7296794.0, + "step": 2700 + }, + { + "entropy": 0.14380223151296379, + "epoch": 6.741559238796808, + "grad_norm": 0.36372771859169006, + "learning_rate": 5.084411085467557e-05, + "loss": 0.09390204429626464, + "mean_token_accuracy": 0.966132538318634, + "num_tokens": 7429156.0, + "step": 2750 + }, + { + "entropy": 0.1392465164512396, + "epoch": 6.864333947206875, + "grad_norm": 0.4069533348083496, + "learning_rate": 4.748066038961626e-05, + "loss": 0.09048093795776367, + "mean_token_accuracy": 0.9678688687086106, + "num_tokens": 7559658.0, + "step": 2800 + }, + { + "entropy": 0.13713553678244353, + "epoch": 6.987108655616943, + "grad_norm": 0.34467506408691406, + "learning_rate": 4.4190379827727676e-05, + "loss": 0.08747495651245117, + "mean_token_accuracy": 0.9685557943582535, + "num_tokens": 7699096.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.22547772577830724, + "eval_loss": 0.8343477249145508, + "eval_mean_token_accuracy": 0.8449994778633118, + "eval_num_tokens": 7712376.0, + "eval_runtime": 105.9168, + "eval_samples_per_second": 13.171, + "eval_steps_per_second": 1.652, + "step": 2856 + }, + { + "entropy": 0.12294510751962662, + "epoch": 7.108041743400859, + "grad_norm": 0.23788245022296906, + "learning_rate": 4.097928923306641e-05, + "loss": 0.07265013217926025, + "mean_token_accuracy": 0.9740867614746094, + "num_tokens": 7825131.0, + "step": 2900 + }, + { + "entropy": 0.11531499886885285, + "epoch": 7.230816451810927, + "grad_norm": 0.23746679723262787, + "learning_rate": 3.785326377971202e-05, + "loss": 0.0625599193572998, + "mean_token_accuracy": 0.9764253079891205, + "num_tokens": 7963208.0, + "step": 2950 + }, + { + "entropy": 0.111452647857368, + "epoch": 7.3535911602209945, + "grad_norm": 0.23431439697742462, + "learning_rate": 3.4818023002250615e-05, + "loss": 0.0625885009765625, + "mean_token_accuracy": 0.9765455546975136, + "num_tokens": 8100530.0, + "step": 3000 + }, + { + "entropy": 0.10402006102725864, + "epoch": 7.476365868631062, + "grad_norm": 0.29770365357398987, + "learning_rate": 3.187912033102466e-05, + "loss": 0.05990049839019775, + "mean_token_accuracy": 0.9781030318140984, + "num_tokens": 8241781.0, + "step": 3050 + }, + { + "entropy": 0.11040891058743, + "epoch": 7.59914057704113, + "grad_norm": 0.32297083735466003, + "learning_rate": 2.9041932931295312e-05, + "loss": 0.06435544013977051, + "mean_token_accuracy": 0.9758345261216164, + "num_tokens": 8374913.0, + "step": 3100 + }, + { + "entropy": 0.11264785576611758, + "epoch": 7.721915285451197, + "grad_norm": 0.37468084692955017, + "learning_rate": 2.6311651864908477e-05, + "loss": 0.06344935417175293, + "mean_token_accuracy": 0.9757431983947754, + "num_tokens": 8511662.0, + "step": 3150 + }, + { + "entropy": 0.11280227556824685, + "epoch": 7.844689993861264, + "grad_norm": 0.2595016658306122, + "learning_rate": 2.3693272592464892e-05, + "loss": 0.06645034790039063, + "mean_token_accuracy": 0.9757053011655807, + "num_tokens": 8645008.0, + "step": 3200 + }, + { + "entropy": 0.1106757521815598, + "epoch": 7.967464702271332, + "grad_norm": 0.3099309206008911, + "learning_rate": 2.1191585833372245e-05, + "loss": 0.06522040367126465, + "mean_token_accuracy": 0.9758621609210968, + "num_tokens": 8778619.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.20031422572476523, + "eval_loss": 0.9309563040733337, + "eval_mean_token_accuracy": 0.8455711133139474, + "eval_num_tokens": 8814144.0, + "eval_runtime": 105.9709, + "eval_samples_per_second": 13.164, + "eval_steps_per_second": 1.651, + "step": 3264 + }, + { + "entropy": 0.10702161500372258, + "epoch": 8.088397790055248, + "grad_norm": 0.29613637924194336, + "learning_rate": 1.8811168800502198e-05, + "loss": 0.05768918514251709, + "mean_token_accuracy": 0.9777823347730685, + "num_tokens": 8907693.0, + "step": 3300 + }, + { + "entropy": 0.10276623284444213, + "epoch": 8.211172498465316, + "grad_norm": 0.3019285798072815, + "learning_rate": 1.6556376825489847e-05, + "loss": 0.05475595474243164, + "mean_token_accuracy": 0.9785520619153977, + "num_tokens": 9041538.0, + "step": 3350 + }, + { + "entropy": 0.1067711115255952, + "epoch": 8.333947206875383, + "grad_norm": 0.1790529489517212, + "learning_rate": 1.4431335389998088e-05, + "loss": 0.05631176471710205, + "mean_token_accuracy": 0.9769352993369103, + "num_tokens": 9173381.0, + "step": 3400 + }, + { + "entropy": 0.10552569851279259, + "epoch": 8.456721915285451, + "grad_norm": 0.14731012284755707, + "learning_rate": 1.243993257752737e-05, + "loss": 0.05628815650939942, + "mean_token_accuracy": 0.9774915510416031, + "num_tokens": 9306859.0, + "step": 3450 + }, + { + "entropy": 0.09718697739765048, + "epoch": 8.579496623695519, + "grad_norm": 0.18447713553905487, + "learning_rate": 1.058581195958119e-05, + "loss": 0.05218749046325684, + "mean_token_accuracy": 0.9795992225408554, + "num_tokens": 9451826.0, + "step": 3500 + }, + { + "entropy": 0.10524881256744266, + "epoch": 8.702271332105585, + "grad_norm": 0.15388962626457214, + "learning_rate": 8.872365929202854e-06, + "loss": 0.05519162178039551, + "mean_token_accuracy": 0.977289491891861, + "num_tokens": 9590049.0, + "step": 3550 + }, + { + "entropy": 0.10105192774906754, + "epoch": 8.825046040515653, + "grad_norm": 0.1201755627989769, + "learning_rate": 7.302729494081316e-06, + "loss": 0.05525091171264648, + "mean_token_accuracy": 0.9778816586732865, + "num_tokens": 9727075.0, + "step": 3600 + }, + { + "entropy": 0.10161143256351352, + "epoch": 8.947820748925722, + "grad_norm": 0.16088183224201202, + "learning_rate": 5.879774540582322e-06, + "loss": 0.05550719261169434, + "mean_token_accuracy": 0.9781533771753311, + "num_tokens": 9862022.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.18716000897543772, + "eval_loss": 1.0105113983154297, + "eval_mean_token_accuracy": 0.8455354629244123, + "eval_num_tokens": 9915912.0, + "eval_runtime": 105.9169, + "eval_samples_per_second": 13.171, + "eval_steps_per_second": 1.652, + "step": 3672 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6545001473251328e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbc706f6b7eb2cc5d17ef1de0d7a89ab44c9683 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.08935122001117755, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..54040305232dfca414bc25478b35259449a47faf --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 408, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4665306943655014, + "epoch": 0.12277470841006753, + "grad_norm": 0.8344842791557312, + "learning_rate": 2.1010394822962346e-05, + "loss": 1.379196014404297, + "mean_token_accuracy": 0.6848452094197274, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.7834547007083893, + "epoch": 0.24554941682013506, + "grad_norm": 1.1248278617858887, + "learning_rate": 4.2449573213740254e-05, + "loss": 0.7345225524902343, + "mean_token_accuracy": 0.7882961884140969, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7194462446868419, + "epoch": 0.3683241252302026, + "grad_norm": 0.6498261094093323, + "learning_rate": 6.388875160451815e-05, + "loss": 0.664264907836914, + "mean_token_accuracy": 0.8042438223958015, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6771699503064156, + "epoch": 0.4910988336402701, + "grad_norm": 0.6653317809104919, + "learning_rate": 8.532792999529606e-05, + "loss": 0.6322076797485352, + "mean_token_accuracy": 0.8130025643110276, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6632554066181183, + "epoch": 0.6138735420503376, + "grad_norm": 0.44293469190597534, + "learning_rate": 0.00010676710838607398, + "loss": 0.617110595703125, + "mean_token_accuracy": 0.8168379080295562, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6508415986597538, + "epoch": 0.7366482504604052, + "grad_norm": 0.49295490980148315, + "learning_rate": 0.00012820628677685187, + "loss": 0.6023779296875, + "mean_token_accuracy": 0.8187075120210647, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6380941693484783, + "epoch": 0.8594229588704727, + "grad_norm": 0.5855488181114197, + "learning_rate": 0.00014964546516762976, + "loss": 0.5912636947631836, + "mean_token_accuracy": 0.8227647486329078, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.6346297286450863, + "epoch": 0.9821976672805403, + "grad_norm": 0.3817698657512665, + "learning_rate": 0.00017108464355840766, + "loss": 0.5902091598510742, + "mean_token_accuracy": 0.8231476756930352, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6600051839011056, + "eval_loss": 0.5857571363449097, + "eval_mean_token_accuracy": 0.8248803649629866, + "eval_num_tokens": 1101768.0, + "eval_runtime": 106.3263, + "eval_samples_per_second": 13.12, + "eval_steps_per_second": 1.646, + "step": 408 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.8417948753232896e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbc706f6b7eb2cc5d17ef1de0d7a89ab44c9683 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.08935122001117755, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3de21c14f5930cc3ef000b3529dfb548296f2ad5 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/trainer_state.json @@ -0,0 +1,954 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4080, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4665306943655014, + "epoch": 0.12277470841006753, + "grad_norm": 0.8344842791557312, + "learning_rate": 2.1010394822962346e-05, + "loss": 1.379196014404297, + "mean_token_accuracy": 0.6848452094197274, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.7834547007083893, + "epoch": 0.24554941682013506, + "grad_norm": 1.1248278617858887, + "learning_rate": 4.2449573213740254e-05, + "loss": 0.7345225524902343, + "mean_token_accuracy": 0.7882961884140969, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7194462446868419, + "epoch": 0.3683241252302026, + "grad_norm": 0.6498261094093323, + "learning_rate": 6.388875160451815e-05, + "loss": 0.664264907836914, + "mean_token_accuracy": 0.8042438223958015, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6771699503064156, + "epoch": 0.4910988336402701, + "grad_norm": 0.6653317809104919, + "learning_rate": 8.532792999529606e-05, + "loss": 0.6322076797485352, + "mean_token_accuracy": 0.8130025643110276, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6632554066181183, + "epoch": 0.6138735420503376, + "grad_norm": 0.44293469190597534, + "learning_rate": 0.00010676710838607398, + "loss": 0.617110595703125, + "mean_token_accuracy": 0.8168379080295562, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6508415986597538, + "epoch": 0.7366482504604052, + "grad_norm": 0.49295490980148315, + "learning_rate": 0.00012820628677685187, + "loss": 0.6023779296875, + "mean_token_accuracy": 0.8187075120210647, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6380941693484783, + "epoch": 0.8594229588704727, + "grad_norm": 0.5855488181114197, + "learning_rate": 0.00014964546516762976, + "loss": 0.5912636947631836, + "mean_token_accuracy": 0.8227647486329078, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.6346297286450863, + "epoch": 0.9821976672805403, + "grad_norm": 0.3817698657512665, + "learning_rate": 0.00017108464355840766, + "loss": 0.5902091598510742, + "mean_token_accuracy": 0.8231476756930352, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6600051839011056, + "eval_loss": 0.5857571363449097, + "eval_mean_token_accuracy": 0.8248803649629866, + "eval_num_tokens": 1101768.0, + "eval_runtime": 106.3263, + "eval_samples_per_second": 13.12, + "eval_steps_per_second": 1.646, + "step": 408 + }, + { + "entropy": 0.5965584144374441, + "epoch": 1.1031307550644567, + "grad_norm": 0.5178472399711609, + "learning_rate": 0.00017488988654901867, + "loss": 0.5578182220458985, + "mean_token_accuracy": 0.830260409921559, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5831904768943786, + "epoch": 1.2259054634745243, + "grad_norm": 0.4383259117603302, + "learning_rate": 0.00017467872603358825, + "loss": 0.5399176788330078, + "mean_token_accuracy": 0.8351450267434121, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5885515750944614, + "epoch": 1.3486801718845918, + "grad_norm": 0.5418892502784729, + "learning_rate": 0.00017430800739444695, + "loss": 0.5402648544311524, + "mean_token_accuracy": 0.8344706323742866, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.603163862079382, + "epoch": 1.4714548802946594, + "grad_norm": 0.5235878825187683, + "learning_rate": 0.0001737784089172037, + "loss": 0.5534031677246094, + "mean_token_accuracy": 0.8304652521014213, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5745155589282512, + "epoch": 1.5942295887047269, + "grad_norm": 0.46109989285469055, + "learning_rate": 0.00017309089958207074, + "loss": 0.5251047134399414, + "mean_token_accuracy": 0.8380702176690101, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5756107035279274, + "epoch": 1.7170042971147943, + "grad_norm": 0.38209712505340576, + "learning_rate": 0.00017224673729096835, + "loss": 0.5253059387207031, + "mean_token_accuracy": 0.8379077833890914, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5695902299880982, + "epoch": 1.839779005524862, + "grad_norm": 0.4063270688056946, + "learning_rate": 0.00017124746656600374, + "loss": 0.5199185180664062, + "mean_token_accuracy": 0.8399301299452782, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5635769453644752, + "epoch": 1.9625537139349294, + "grad_norm": 0.4165436029434204, + "learning_rate": 0.00017009491572353503, + "loss": 0.5156089019775391, + "mean_token_accuracy": 0.8404546654224396, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5587481154714312, + "eval_loss": 0.5402795672416687, + "eval_mean_token_accuracy": 0.8386626076698304, + "eval_num_tokens": 2203536.0, + "eval_runtime": 105.9883, + "eval_samples_per_second": 13.162, + "eval_steps_per_second": 1.651, + "step": 816 + }, + { + "entropy": 0.5002549305785126, + "epoch": 2.0834868017188457, + "grad_norm": 0.4061429500579834, + "learning_rate": 0.00016879119352899107, + "loss": 0.4499735641479492, + "mean_token_accuracy": 0.8566478448470837, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.4894501394033432, + "epoch": 2.2062615101289134, + "grad_norm": 0.530322790145874, + "learning_rate": 0.00016733868533856737, + "loss": 0.4376458740234375, + "mean_token_accuracy": 0.859256779551506, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.48188683822751044, + "epoch": 2.329036218538981, + "grad_norm": 0.4338841438293457, + "learning_rate": 0.0001657400487348575, + "loss": 0.435361328125, + "mean_token_accuracy": 0.8610378202795982, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.4992289599776268, + "epoch": 2.4518109269490487, + "grad_norm": 0.44097110629081726, + "learning_rate": 0.00016399820866440517, + "loss": 0.4474017333984375, + "mean_token_accuracy": 0.8565557089447975, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.49334951683878897, + "epoch": 2.574585635359116, + "grad_norm": 0.48084133863449097, + "learning_rate": 0.00016211635208607405, + "loss": 0.440751953125, + "mean_token_accuracy": 0.8580261027812958, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.4898635870218277, + "epoch": 2.6973603437691835, + "grad_norm": 0.40215447545051575, + "learning_rate": 0.00016009792214002617, + "loss": 0.44588775634765626, + "mean_token_accuracy": 0.8572473660111427, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.4932038363814354, + "epoch": 2.820135052179251, + "grad_norm": 0.4465612769126892, + "learning_rate": 0.00015794661184797816, + "loss": 0.4444055938720703, + "mean_token_accuracy": 0.8583759877085686, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.4850301502645016, + "epoch": 2.942909760589319, + "grad_norm": 0.43597331643104553, + "learning_rate": 0.0001556663573562617, + "loss": 0.4382463073730469, + "mean_token_accuracy": 0.8603282216191291, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.4902002043383462, + "eval_loss": 0.5276822447776794, + "eval_mean_token_accuracy": 0.8454030500139509, + "eval_num_tokens": 3305304.0, + "eval_runtime": 106.004, + "eval_samples_per_second": 13.16, + "eval_steps_per_second": 1.651, + "step": 1224 + }, + { + "entropy": 0.4309167733349776, + "epoch": 3.063842848373235, + "grad_norm": 0.4111804664134979, + "learning_rate": 0.00015326133073405068, + "loss": 0.3837854766845703, + "mean_token_accuracy": 0.8741549949355537, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.38693042084574697, + "epoch": 3.1866175567833026, + "grad_norm": 0.46740710735321045, + "learning_rate": 0.00015073593233993186, + "loss": 0.335601806640625, + "mean_token_accuracy": 0.8861287096142769, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.39316988229751587, + "epoch": 3.3093922651933703, + "grad_norm": 0.5308303236961365, + "learning_rate": 0.00014809478277078602, + "loss": 0.34588619232177736, + "mean_token_accuracy": 0.8837816697359085, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.39958367675542833, + "epoch": 3.4321669736034375, + "grad_norm": 0.5955453515052795, + "learning_rate": 0.00014534271440770966, + "loss": 0.34634483337402344, + "mean_token_accuracy": 0.8832473835349083, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.4100498601794243, + "epoch": 3.554941682013505, + "grad_norm": 0.4593118727207184, + "learning_rate": 0.0001424847625744456, + "loss": 0.3560930633544922, + "mean_token_accuracy": 0.8804642978310585, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.39962531872093676, + "epoch": 3.677716390423573, + "grad_norm": 0.4584615230560303, + "learning_rate": 0.00013952615632449963, + "loss": 0.34978450775146486, + "mean_token_accuracy": 0.8825609818100929, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.39326138779520986, + "epoch": 3.80049109883364, + "grad_norm": 0.5227728486061096, + "learning_rate": 0.00013647230887379902, + "loss": 0.3470985794067383, + "mean_token_accuracy": 0.8833131140470505, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.399793985709548, + "epoch": 3.9232658072437077, + "grad_norm": 0.4263511002063751, + "learning_rate": 0.00013332880769639856, + "loss": 0.3550622177124023, + "mean_token_accuracy": 0.8825028780102729, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.41604454040527344, + "eval_loss": 0.5501811504364014, + "eval_mean_token_accuracy": 0.8466314860752651, + "eval_num_tokens": 4407072.0, + "eval_runtime": 105.9531, + "eval_samples_per_second": 13.166, + "eval_steps_per_second": 1.652, + "step": 1632 + }, + { + "entropy": 0.3492097145107192, + "epoch": 4.044198895027624, + "grad_norm": 0.41719651222229004, + "learning_rate": 0.00013010140430135466, + "loss": 0.30388387680053713, + "mean_token_accuracy": 0.8972433387930623, + "num_tokens": 4455549.0, + "step": 1650 + }, + { + "entropy": 0.2980811195075512, + "epoch": 4.166973603437691, + "grad_norm": 0.694450318813324, + "learning_rate": 0.00012679600370947314, + "loss": 0.24677230834960937, + "mean_token_accuracy": 0.9134943392872811, + "num_tokens": 4584218.0, + "step": 1700 + }, + { + "entropy": 0.2911450309306383, + "epoch": 4.2897483118477595, + "grad_norm": 0.4893203675746918, + "learning_rate": 0.00012341865364918372, + "loss": 0.24411561965942383, + "mean_token_accuracy": 0.9145414170622825, + "num_tokens": 4724425.0, + "step": 1750 + }, + { + "entropy": 0.2982170185446739, + "epoch": 4.412523020257827, + "grad_norm": 0.5446121096611023, + "learning_rate": 0.00011997553349130966, + "loss": 0.24687444686889648, + "mean_token_accuracy": 0.9126561579108238, + "num_tokens": 4861552.0, + "step": 1800 + }, + { + "entropy": 0.30126186162233354, + "epoch": 4.535297728667894, + "grad_norm": 0.5465192198753357, + "learning_rate": 0.00011647294294297787, + "loss": 0.2527424621582031, + "mean_token_accuracy": 0.9110122618079185, + "num_tokens": 4993610.0, + "step": 1850 + }, + { + "entropy": 0.2993039993941784, + "epoch": 4.658072437077962, + "grad_norm": 0.6376634836196899, + "learning_rate": 0.00011291729052135527, + "loss": 0.2507955169677734, + "mean_token_accuracy": 0.9124656105041504, + "num_tokens": 5126291.0, + "step": 1900 + }, + { + "entropy": 0.29630616143345834, + "epoch": 4.780847145488029, + "grad_norm": 0.5191586017608643, + "learning_rate": 0.00010931508182830125, + "loss": 0.2503428077697754, + "mean_token_accuracy": 0.9127612486481667, + "num_tokens": 5266076.0, + "step": 1950 + }, + { + "entropy": 0.29707231901586056, + "epoch": 4.903621853898097, + "grad_norm": 0.42329415678977966, + "learning_rate": 0.00010567290764738874, + "loss": 0.24930551528930664, + "mean_token_accuracy": 0.911863605082035, + "num_tokens": 5403761.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.34850709250995093, + "eval_loss": 0.6053183078765869, + "eval_mean_token_accuracy": 0.8461176470347813, + "eval_num_tokens": 5508840.0, + "eval_runtime": 105.9653, + "eval_samples_per_second": 13.165, + "eval_steps_per_second": 1.651, + "step": 2040 + }, + { + "entropy": 0.28518860919523964, + "epoch": 5.024554941682013, + "grad_norm": 0.6316563487052917, + "learning_rate": 0.00010199743188507247, + "loss": 0.23341741561889648, + "mean_token_accuracy": 0.9183166952907736, + "num_tokens": 5535743.0, + "step": 2050 + }, + { + "entropy": 0.20185263987630606, + "epoch": 5.147329650092081, + "grad_norm": 0.6542733311653137, + "learning_rate": 9.829537937806829e-05, + "loss": 0.1496055793762207, + "mean_token_accuracy": 0.9459337556362152, + "num_tokens": 5675294.0, + "step": 2100 + }, + { + "entropy": 0.2101737355440855, + "epoch": 5.270104358502149, + "grad_norm": 0.46600785851478577, + "learning_rate": 9.457352358925075e-05, + "loss": 0.15642745018005372, + "mean_token_accuracy": 0.944338481426239, + "num_tokens": 5808279.0, + "step": 2150 + }, + { + "entropy": 0.200700718536973, + "epoch": 5.392879066912216, + "grad_norm": 0.5562126636505127, + "learning_rate": 9.083867421458257e-05, + "loss": 0.15122636795043945, + "mean_token_accuracy": 0.9447117391228675, + "num_tokens": 5948774.0, + "step": 2200 + }, + { + "entropy": 0.2139152016490698, + "epoch": 5.515653775322283, + "grad_norm": 0.5136730670928955, + "learning_rate": 8.709766472374944e-05, + "loss": 0.16092844009399415, + "mean_token_accuracy": 0.9423242849111557, + "num_tokens": 6081211.0, + "step": 2250 + }, + { + "entropy": 0.20662450045347214, + "epoch": 5.638428483732351, + "grad_norm": 0.6007277965545654, + "learning_rate": 8.335733985729791e-05, + "loss": 0.15495609283447265, + "mean_token_accuracy": 0.9431806969642639, + "num_tokens": 6213793.0, + "step": 2300 + }, + { + "entropy": 0.20562831707298757, + "epoch": 5.7612031921424185, + "grad_norm": 0.5216547846794128, + "learning_rate": 7.962454310315095e-05, + "loss": 0.1568380641937256, + "mean_token_accuracy": 0.9425676354765892, + "num_tokens": 6350388.0, + "step": 2350 + }, + { + "entropy": 0.20440548222512006, + "epoch": 5.883977900552486, + "grad_norm": 0.5724782943725586, + "learning_rate": 7.590610417541581e-05, + "loss": 0.15627790451049806, + "mean_token_accuracy": 0.9437827947735786, + "num_tokens": 6485960.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.2856926244497299, + "eval_loss": 0.7071177959442139, + "eval_mean_token_accuracy": 0.8432521397726876, + "eval_num_tokens": 6610608.0, + "eval_runtime": 105.9542, + "eval_samples_per_second": 13.166, + "eval_steps_per_second": 1.652, + "step": 2448 + }, + { + "entropy": 0.20338259523895186, + "epoch": 6.004910988336403, + "grad_norm": 0.39025962352752686, + "learning_rate": 7.220882651839295e-05, + "loss": 0.15414920806884766, + "mean_token_accuracy": 0.9444228456105073, + "num_tokens": 6616117.0, + "step": 2450 + }, + { + "entropy": 0.13960296589881183, + "epoch": 6.12768569674647, + "grad_norm": 0.4555230438709259, + "learning_rate": 6.85394748586495e-05, + "loss": 0.08990025520324707, + "mean_token_accuracy": 0.968256405889988, + "num_tokens": 6754569.0, + "step": 2500 + }, + { + "entropy": 0.13907674692571162, + "epoch": 6.250460405156538, + "grad_norm": 0.49845439195632935, + "learning_rate": 6.490476282793282e-05, + "loss": 0.08988103866577149, + "mean_token_accuracy": 0.9675386163592339, + "num_tokens": 6893447.0, + "step": 2550 + }, + { + "entropy": 0.143365166708827, + "epoch": 6.373235113566605, + "grad_norm": 0.5541622042655945, + "learning_rate": 6.131134067956953e-05, + "loss": 0.09036359786987305, + "mean_token_accuracy": 0.9669707071781158, + "num_tokens": 7025939.0, + "step": 2600 + }, + { + "entropy": 0.14202470190823077, + "epoch": 6.496009821976672, + "grad_norm": 0.38522830605506897, + "learning_rate": 5.776578312082506e-05, + "loss": 0.08982279777526855, + "mean_token_accuracy": 0.9668073603510856, + "num_tokens": 7163978.0, + "step": 2650 + }, + { + "entropy": 0.14029258273541928, + "epoch": 6.6187845303867405, + "grad_norm": 0.4234164357185364, + "learning_rate": 5.4274577283486146e-05, + "loss": 0.09312637329101563, + "mean_token_accuracy": 0.9669712704420089, + "num_tokens": 7296794.0, + "step": 2700 + }, + { + "entropy": 0.14380223151296379, + "epoch": 6.741559238796808, + "grad_norm": 0.36372771859169006, + "learning_rate": 5.084411085467557e-05, + "loss": 0.09390204429626464, + "mean_token_accuracy": 0.966132538318634, + "num_tokens": 7429156.0, + "step": 2750 + }, + { + "entropy": 0.1392465164512396, + "epoch": 6.864333947206875, + "grad_norm": 0.4069533348083496, + "learning_rate": 4.748066038961626e-05, + "loss": 0.09048093795776367, + "mean_token_accuracy": 0.9678688687086106, + "num_tokens": 7559658.0, + "step": 2800 + }, + { + "entropy": 0.13713553678244353, + "epoch": 6.987108655616943, + "grad_norm": 0.34467506408691406, + "learning_rate": 4.4190379827727676e-05, + "loss": 0.08747495651245117, + "mean_token_accuracy": 0.9685557943582535, + "num_tokens": 7699096.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.22547772577830724, + "eval_loss": 0.8343477249145508, + "eval_mean_token_accuracy": 0.8449994778633118, + "eval_num_tokens": 7712376.0, + "eval_runtime": 105.9168, + "eval_samples_per_second": 13.171, + "eval_steps_per_second": 1.652, + "step": 2856 + }, + { + "entropy": 0.12294510751962662, + "epoch": 7.108041743400859, + "grad_norm": 0.23788245022296906, + "learning_rate": 4.097928923306641e-05, + "loss": 0.07265013217926025, + "mean_token_accuracy": 0.9740867614746094, + "num_tokens": 7825131.0, + "step": 2900 + }, + { + "entropy": 0.11531499886885285, + "epoch": 7.230816451810927, + "grad_norm": 0.23746679723262787, + "learning_rate": 3.785326377971202e-05, + "loss": 0.0625599193572998, + "mean_token_accuracy": 0.9764253079891205, + "num_tokens": 7963208.0, + "step": 2950 + }, + { + "entropy": 0.111452647857368, + "epoch": 7.3535911602209945, + "grad_norm": 0.23431439697742462, + "learning_rate": 3.4818023002250615e-05, + "loss": 0.0625885009765625, + "mean_token_accuracy": 0.9765455546975136, + "num_tokens": 8100530.0, + "step": 3000 + }, + { + "entropy": 0.10402006102725864, + "epoch": 7.476365868631062, + "grad_norm": 0.29770365357398987, + "learning_rate": 3.187912033102466e-05, + "loss": 0.05990049839019775, + "mean_token_accuracy": 0.9781030318140984, + "num_tokens": 8241781.0, + "step": 3050 + }, + { + "entropy": 0.11040891058743, + "epoch": 7.59914057704113, + "grad_norm": 0.32297083735466003, + "learning_rate": 2.9041932931295312e-05, + "loss": 0.06435544013977051, + "mean_token_accuracy": 0.9758345261216164, + "num_tokens": 8374913.0, + "step": 3100 + }, + { + "entropy": 0.11264785576611758, + "epoch": 7.721915285451197, + "grad_norm": 0.37468084692955017, + "learning_rate": 2.6311651864908477e-05, + "loss": 0.06344935417175293, + "mean_token_accuracy": 0.9757431983947754, + "num_tokens": 8511662.0, + "step": 3150 + }, + { + "entropy": 0.11280227556824685, + "epoch": 7.844689993861264, + "grad_norm": 0.2595016658306122, + "learning_rate": 2.3693272592464892e-05, + "loss": 0.06645034790039063, + "mean_token_accuracy": 0.9757053011655807, + "num_tokens": 8645008.0, + "step": 3200 + }, + { + "entropy": 0.1106757521815598, + "epoch": 7.967464702271332, + "grad_norm": 0.3099309206008911, + "learning_rate": 2.1191585833372245e-05, + "loss": 0.06522040367126465, + "mean_token_accuracy": 0.9758621609210968, + "num_tokens": 8778619.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.20031422572476523, + "eval_loss": 0.9309563040733337, + "eval_mean_token_accuracy": 0.8455711133139474, + "eval_num_tokens": 8814144.0, + "eval_runtime": 105.9709, + "eval_samples_per_second": 13.164, + "eval_steps_per_second": 1.651, + "step": 3264 + }, + { + "entropy": 0.10702161500372258, + "epoch": 8.088397790055248, + "grad_norm": 0.29613637924194336, + "learning_rate": 1.8811168800502198e-05, + "loss": 0.05768918514251709, + "mean_token_accuracy": 0.9777823347730685, + "num_tokens": 8907693.0, + "step": 3300 + }, + { + "entropy": 0.10276623284444213, + "epoch": 8.211172498465316, + "grad_norm": 0.3019285798072815, + "learning_rate": 1.6556376825489847e-05, + "loss": 0.05475595474243164, + "mean_token_accuracy": 0.9785520619153977, + "num_tokens": 9041538.0, + "step": 3350 + }, + { + "entropy": 0.1067711115255952, + "epoch": 8.333947206875383, + "grad_norm": 0.1790529489517212, + "learning_rate": 1.4431335389998088e-05, + "loss": 0.05631176471710205, + "mean_token_accuracy": 0.9769352993369103, + "num_tokens": 9173381.0, + "step": 3400 + }, + { + "entropy": 0.10552569851279259, + "epoch": 8.456721915285451, + "grad_norm": 0.14731012284755707, + "learning_rate": 1.243993257752737e-05, + "loss": 0.05628815650939942, + "mean_token_accuracy": 0.9774915510416031, + "num_tokens": 9306859.0, + "step": 3450 + }, + { + "entropy": 0.09718697739765048, + "epoch": 8.579496623695519, + "grad_norm": 0.18447713553905487, + "learning_rate": 1.058581195958119e-05, + "loss": 0.05218749046325684, + "mean_token_accuracy": 0.9795992225408554, + "num_tokens": 9451826.0, + "step": 3500 + }, + { + "entropy": 0.10524881256744266, + "epoch": 8.702271332105585, + "grad_norm": 0.15388962626457214, + "learning_rate": 8.872365929202854e-06, + "loss": 0.05519162178039551, + "mean_token_accuracy": 0.977289491891861, + "num_tokens": 9590049.0, + "step": 3550 + }, + { + "entropy": 0.10105192774906754, + "epoch": 8.825046040515653, + "grad_norm": 0.1201755627989769, + "learning_rate": 7.302729494081316e-06, + "loss": 0.05525091171264648, + "mean_token_accuracy": 0.9778816586732865, + "num_tokens": 9727075.0, + "step": 3600 + }, + { + "entropy": 0.10161143256351352, + "epoch": 8.947820748925722, + "grad_norm": 0.16088183224201202, + "learning_rate": 5.879774540582322e-06, + "loss": 0.05550719261169434, + "mean_token_accuracy": 0.9781533771753311, + "num_tokens": 9862022.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.18716000897543772, + "eval_loss": 1.0105113983154297, + "eval_mean_token_accuracy": 0.8455354629244123, + "eval_num_tokens": 9915912.0, + "eval_runtime": 105.9169, + "eval_samples_per_second": 13.171, + "eval_steps_per_second": 1.652, + "step": 3672 + }, + { + "entropy": 0.0988658586999184, + "epoch": 9.068753836709638, + "grad_norm": 0.149685338139534, + "learning_rate": 4.60610457919947e-06, + "loss": 0.05366872310638428, + "mean_token_accuracy": 0.9793282523373057, + "num_tokens": 9994836.0, + "step": 3700 + }, + { + "entropy": 0.10369110053405166, + "epoch": 9.191528545119706, + "grad_norm": 0.1696028858423233, + "learning_rate": 3.484049981039496e-06, + "loss": 0.05339935302734375, + "mean_token_accuracy": 0.9786092588305473, + "num_tokens": 10125546.0, + "step": 3750 + }, + { + "entropy": 0.10147308373823762, + "epoch": 9.314303253529772, + "grad_norm": 0.12892360985279083, + "learning_rate": 2.515663714057214e-06, + "loss": 0.05331225872039795, + "mean_token_accuracy": 0.9789802286028862, + "num_tokens": 10257820.0, + "step": 3800 + }, + { + "entropy": 0.09684176241979003, + "epoch": 9.43707796193984, + "grad_norm": 0.1375845968723297, + "learning_rate": 1.7027175868413575e-06, + "loss": 0.05270596027374268, + "mean_token_accuracy": 0.9791027170419693, + "num_tokens": 10394276.0, + "step": 3850 + }, + { + "entropy": 0.10605986172333359, + "epoch": 9.559852670349908, + "grad_norm": 0.13426834344863892, + "learning_rate": 1.0466990068236981e-06, + "loss": 0.05455333709716797, + "mean_token_accuracy": 0.9776863658428192, + "num_tokens": 10525083.0, + "step": 3900 + }, + { + "entropy": 0.09641905350610613, + "epoch": 9.682627378759975, + "grad_norm": 0.11037543416023254, + "learning_rate": 5.48808258843089e-07, + "loss": 0.051173315048217774, + "mean_token_accuracy": 0.979258199930191, + "num_tokens": 10662496.0, + "step": 3950 + }, + { + "entropy": 0.09776237597689033, + "epoch": 9.805402087170043, + "grad_norm": 0.11856469511985779, + "learning_rate": 2.0995630904351996e-07, + "loss": 0.0521266508102417, + "mean_token_accuracy": 0.9795272997021676, + "num_tokens": 10799808.0, + "step": 4000 + }, + { + "entropy": 0.09680028831586242, + "epoch": 9.92817679558011, + "grad_norm": 0.1150144636631012, + "learning_rate": 3.0763138124227917e-08, + "loss": 0.05109185695648193, + "mean_token_accuracy": 0.9797486981749535, + "num_tokens": 10939770.0, + "step": 4050 + }, + { + "epoch": 10.0, + "eval_entropy": 0.18078699486596245, + "eval_loss": 1.0454745292663574, + "eval_mean_token_accuracy": 0.8454314739363534, + "eval_num_tokens": 11017680.0, + "eval_runtime": 105.8921, + "eval_samples_per_second": 13.174, + "eval_steps_per_second": 1.653, + "step": 4080 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.8381974283200102e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbc706f6b7eb2cc5d17ef1de0d7a89ab44c9683 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.08935122001117755, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8b7fddd633f231ce77d8104ddbc897cd238e1ecd --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/trainer_state.json @@ -0,0 +1,216 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 816, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4665306943655014, + "epoch": 0.12277470841006753, + "grad_norm": 0.8344842791557312, + "learning_rate": 2.1010394822962346e-05, + "loss": 1.379196014404297, + "mean_token_accuracy": 0.6848452094197274, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.7834547007083893, + "epoch": 0.24554941682013506, + "grad_norm": 1.1248278617858887, + "learning_rate": 4.2449573213740254e-05, + "loss": 0.7345225524902343, + "mean_token_accuracy": 0.7882961884140969, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7194462446868419, + "epoch": 0.3683241252302026, + "grad_norm": 0.6498261094093323, + "learning_rate": 6.388875160451815e-05, + "loss": 0.664264907836914, + "mean_token_accuracy": 0.8042438223958015, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6771699503064156, + "epoch": 0.4910988336402701, + "grad_norm": 0.6653317809104919, + "learning_rate": 8.532792999529606e-05, + "loss": 0.6322076797485352, + "mean_token_accuracy": 0.8130025643110276, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6632554066181183, + "epoch": 0.6138735420503376, + "grad_norm": 0.44293469190597534, + "learning_rate": 0.00010676710838607398, + "loss": 0.617110595703125, + "mean_token_accuracy": 0.8168379080295562, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6508415986597538, + "epoch": 0.7366482504604052, + "grad_norm": 0.49295490980148315, + "learning_rate": 0.00012820628677685187, + "loss": 0.6023779296875, + "mean_token_accuracy": 0.8187075120210647, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6380941693484783, + "epoch": 0.8594229588704727, + "grad_norm": 0.5855488181114197, + "learning_rate": 0.00014964546516762976, + "loss": 0.5912636947631836, + "mean_token_accuracy": 0.8227647486329078, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.6346297286450863, + "epoch": 0.9821976672805403, + "grad_norm": 0.3817698657512665, + "learning_rate": 0.00017108464355840766, + "loss": 0.5902091598510742, + "mean_token_accuracy": 0.8231476756930352, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6600051839011056, + "eval_loss": 0.5857571363449097, + "eval_mean_token_accuracy": 0.8248803649629866, + "eval_num_tokens": 1101768.0, + "eval_runtime": 106.3263, + "eval_samples_per_second": 13.12, + "eval_steps_per_second": 1.646, + "step": 408 + }, + { + "entropy": 0.5965584144374441, + "epoch": 1.1031307550644567, + "grad_norm": 0.5178472399711609, + "learning_rate": 0.00017488988654901867, + "loss": 0.5578182220458985, + "mean_token_accuracy": 0.830260409921559, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5831904768943786, + "epoch": 1.2259054634745243, + "grad_norm": 0.4383259117603302, + "learning_rate": 0.00017467872603358825, + "loss": 0.5399176788330078, + "mean_token_accuracy": 0.8351450267434121, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5885515750944614, + "epoch": 1.3486801718845918, + "grad_norm": 0.5418892502784729, + "learning_rate": 0.00017430800739444695, + "loss": 0.5402648544311524, + "mean_token_accuracy": 0.8344706323742866, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.603163862079382, + "epoch": 1.4714548802946594, + "grad_norm": 0.5235878825187683, + "learning_rate": 0.0001737784089172037, + "loss": 0.5534031677246094, + "mean_token_accuracy": 0.8304652521014213, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5745155589282512, + "epoch": 1.5942295887047269, + "grad_norm": 0.46109989285469055, + "learning_rate": 0.00017309089958207074, + "loss": 0.5251047134399414, + "mean_token_accuracy": 0.8380702176690101, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5756107035279274, + "epoch": 1.7170042971147943, + "grad_norm": 0.38209712505340576, + "learning_rate": 0.00017224673729096835, + "loss": 0.5253059387207031, + "mean_token_accuracy": 0.8379077833890914, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5695902299880982, + "epoch": 1.839779005524862, + "grad_norm": 0.4063270688056946, + "learning_rate": 0.00017124746656600374, + "loss": 0.5199185180664062, + "mean_token_accuracy": 0.8399301299452782, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5635769453644752, + "epoch": 1.9625537139349294, + "grad_norm": 0.4165436029434204, + "learning_rate": 0.00017009491572353503, + "loss": 0.5156089019775391, + "mean_token_accuracy": 0.8404546654224396, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5587481154714312, + "eval_loss": 0.5402795672416687, + "eval_mean_token_accuracy": 0.8386626076698304, + "eval_num_tokens": 2203536.0, + "eval_runtime": 105.9883, + "eval_samples_per_second": 13.162, + "eval_steps_per_second": 1.651, + "step": 816 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.679528285369037e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}