diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3d69e66b78ef4bc93e606b70637f654b112f2d6c --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: transformers +model_name: Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1 +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test1 + +This model is a fine-tuned version of [Qwen/Qwen3-14B-Base](https://huggingface.co/Qwen/Qwen3-14B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/r2wxxlcd) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cee4168b05f186b96700372be25fed4836d8e734 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: transformers +model_name: Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2 +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2 + +This model is a fine-tuned version of [Qwen/Qwen3-14B-Base](https://huggingface.co/Qwen/Qwen3-14B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/daxtch08) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbc706f6b7eb2cc5d17ef1de0d7a89ab44c9683 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.08935122001117755, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ea66c7ff86fdbee850d89ff0c3d5c28984558fc2 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/trainer_state.json @@ -0,0 +1,307 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1224, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4665306943655014, + "epoch": 0.12277470841006753, + "grad_norm": 0.8344842791557312, + "learning_rate": 2.1010394822962346e-05, + "loss": 1.379196014404297, + "mean_token_accuracy": 0.6848452094197274, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.7834547007083893, + "epoch": 0.24554941682013506, + "grad_norm": 1.1248278617858887, + "learning_rate": 4.2449573213740254e-05, + "loss": 0.7345225524902343, + "mean_token_accuracy": 0.7882961884140969, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7194462446868419, + "epoch": 0.3683241252302026, + "grad_norm": 0.6498261094093323, + "learning_rate": 6.388875160451815e-05, + "loss": 0.664264907836914, + "mean_token_accuracy": 0.8042438223958015, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6771699503064156, + "epoch": 0.4910988336402701, + "grad_norm": 0.6653317809104919, + "learning_rate": 8.532792999529606e-05, + "loss": 0.6322076797485352, + "mean_token_accuracy": 0.8130025643110276, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6632554066181183, + "epoch": 0.6138735420503376, + "grad_norm": 0.44293469190597534, + "learning_rate": 0.00010676710838607398, + "loss": 0.617110595703125, + "mean_token_accuracy": 0.8168379080295562, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6508415986597538, + "epoch": 0.7366482504604052, + "grad_norm": 0.49295490980148315, + "learning_rate": 0.00012820628677685187, + "loss": 0.6023779296875, + "mean_token_accuracy": 0.8187075120210647, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6380941693484783, + "epoch": 0.8594229588704727, + "grad_norm": 0.5855488181114197, + "learning_rate": 0.00014964546516762976, + "loss": 0.5912636947631836, + "mean_token_accuracy": 0.8227647486329078, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.6346297286450863, + "epoch": 0.9821976672805403, + "grad_norm": 0.3817698657512665, + "learning_rate": 0.00017108464355840766, + "loss": 0.5902091598510742, + "mean_token_accuracy": 0.8231476756930352, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6600051839011056, + "eval_loss": 0.5857571363449097, + "eval_mean_token_accuracy": 0.8248803649629866, + "eval_num_tokens": 1101768.0, + "eval_runtime": 106.3263, + "eval_samples_per_second": 13.12, + "eval_steps_per_second": 1.646, + "step": 408 + }, + { + "entropy": 0.5965584144374441, + "epoch": 1.1031307550644567, + "grad_norm": 0.5178472399711609, + "learning_rate": 0.00017488988654901867, + "loss": 0.5578182220458985, + "mean_token_accuracy": 0.830260409921559, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5831904768943786, + "epoch": 1.2259054634745243, + "grad_norm": 0.4383259117603302, + "learning_rate": 0.00017467872603358825, + "loss": 0.5399176788330078, + "mean_token_accuracy": 0.8351450267434121, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5885515750944614, + "epoch": 1.3486801718845918, + "grad_norm": 0.5418892502784729, + "learning_rate": 0.00017430800739444695, + "loss": 0.5402648544311524, + "mean_token_accuracy": 0.8344706323742866, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.603163862079382, + "epoch": 1.4714548802946594, + "grad_norm": 0.5235878825187683, + "learning_rate": 0.0001737784089172037, + "loss": 0.5534031677246094, + "mean_token_accuracy": 0.8304652521014213, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5745155589282512, + "epoch": 1.5942295887047269, + "grad_norm": 0.46109989285469055, + "learning_rate": 0.00017309089958207074, + "loss": 0.5251047134399414, + "mean_token_accuracy": 0.8380702176690101, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5756107035279274, + "epoch": 1.7170042971147943, + "grad_norm": 0.38209712505340576, + "learning_rate": 0.00017224673729096835, + "loss": 0.5253059387207031, + "mean_token_accuracy": 0.8379077833890914, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5695902299880982, + "epoch": 1.839779005524862, + "grad_norm": 0.4063270688056946, + "learning_rate": 0.00017124746656600374, + "loss": 0.5199185180664062, + "mean_token_accuracy": 0.8399301299452782, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5635769453644752, + "epoch": 1.9625537139349294, + "grad_norm": 0.4165436029434204, + "learning_rate": 0.00017009491572353503, + "loss": 0.5156089019775391, + "mean_token_accuracy": 0.8404546654224396, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5587481154714312, + "eval_loss": 0.5402795672416687, + "eval_mean_token_accuracy": 0.8386626076698304, + "eval_num_tokens": 2203536.0, + "eval_runtime": 105.9883, + "eval_samples_per_second": 13.162, + "eval_steps_per_second": 1.651, + "step": 816 + }, + { + "entropy": 0.5002549305785126, + "epoch": 2.0834868017188457, + "grad_norm": 0.4061429500579834, + "learning_rate": 0.00016879119352899107, + "loss": 0.4499735641479492, + "mean_token_accuracy": 0.8566478448470837, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.4894501394033432, + "epoch": 2.2062615101289134, + "grad_norm": 0.530322790145874, + "learning_rate": 0.00016733868533856737, + "loss": 0.4376458740234375, + "mean_token_accuracy": 0.859256779551506, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.48188683822751044, + "epoch": 2.329036218538981, + "grad_norm": 0.4338841438293457, + "learning_rate": 0.0001657400487348575, + "loss": 0.435361328125, + "mean_token_accuracy": 0.8610378202795982, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.4992289599776268, + "epoch": 2.4518109269490487, + "grad_norm": 0.44097110629081726, + "learning_rate": 0.00016399820866440517, + "loss": 0.4474017333984375, + "mean_token_accuracy": 0.8565557089447975, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.49334951683878897, + "epoch": 2.574585635359116, + "grad_norm": 0.48084133863449097, + "learning_rate": 0.00016211635208607405, + "loss": 0.440751953125, + "mean_token_accuracy": 0.8580261027812958, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.4898635870218277, + "epoch": 2.6973603437691835, + "grad_norm": 0.40215447545051575, + "learning_rate": 0.00016009792214002617, + "loss": 0.44588775634765626, + "mean_token_accuracy": 0.8572473660111427, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.4932038363814354, + "epoch": 2.820135052179251, + "grad_norm": 0.4465612769126892, + "learning_rate": 0.00015794661184797816, + "loss": 0.4444055938720703, + "mean_token_accuracy": 0.8583759877085686, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.4850301502645016, + "epoch": 2.942909760589319, + "grad_norm": 0.43597331643104553, + "learning_rate": 0.0001556663573562617, + "loss": 0.4382463073730469, + "mean_token_accuracy": 0.8603282216191291, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.4902002043383462, + "eval_loss": 0.5276822447776794, + "eval_mean_token_accuracy": 0.8454030500139509, + "eval_num_tokens": 3305304.0, + "eval_runtime": 106.004, + "eval_samples_per_second": 13.16, + "eval_steps_per_second": 1.651, + "step": 1224 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.5230462593101824e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbc706f6b7eb2cc5d17ef1de0d7a89ab44c9683 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.08935122001117755, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..50a58630dd206c20385d6fbc2cded84002f5c778 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/trainer_state.json @@ -0,0 +1,398 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1632, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4665306943655014, + "epoch": 0.12277470841006753, + "grad_norm": 0.8344842791557312, + "learning_rate": 2.1010394822962346e-05, + "loss": 1.379196014404297, + "mean_token_accuracy": 0.6848452094197274, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.7834547007083893, + "epoch": 0.24554941682013506, + "grad_norm": 1.1248278617858887, + "learning_rate": 4.2449573213740254e-05, + "loss": 0.7345225524902343, + "mean_token_accuracy": 0.7882961884140969, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7194462446868419, + "epoch": 0.3683241252302026, + "grad_norm": 0.6498261094093323, + "learning_rate": 6.388875160451815e-05, + "loss": 0.664264907836914, + "mean_token_accuracy": 0.8042438223958015, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6771699503064156, + "epoch": 0.4910988336402701, + "grad_norm": 0.6653317809104919, + "learning_rate": 8.532792999529606e-05, + "loss": 0.6322076797485352, + "mean_token_accuracy": 0.8130025643110276, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6632554066181183, + "epoch": 0.6138735420503376, + "grad_norm": 0.44293469190597534, + "learning_rate": 0.00010676710838607398, + "loss": 0.617110595703125, + "mean_token_accuracy": 0.8168379080295562, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6508415986597538, + "epoch": 0.7366482504604052, + "grad_norm": 0.49295490980148315, + "learning_rate": 0.00012820628677685187, + "loss": 0.6023779296875, + "mean_token_accuracy": 0.8187075120210647, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6380941693484783, + "epoch": 0.8594229588704727, + "grad_norm": 0.5855488181114197, + "learning_rate": 0.00014964546516762976, + "loss": 0.5912636947631836, + "mean_token_accuracy": 0.8227647486329078, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.6346297286450863, + "epoch": 0.9821976672805403, + "grad_norm": 0.3817698657512665, + "learning_rate": 0.00017108464355840766, + "loss": 0.5902091598510742, + "mean_token_accuracy": 0.8231476756930352, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6600051839011056, + "eval_loss": 0.5857571363449097, + "eval_mean_token_accuracy": 0.8248803649629866, + "eval_num_tokens": 1101768.0, + "eval_runtime": 106.3263, + "eval_samples_per_second": 13.12, + "eval_steps_per_second": 1.646, + "step": 408 + }, + { + "entropy": 0.5965584144374441, + "epoch": 1.1031307550644567, + "grad_norm": 0.5178472399711609, + "learning_rate": 0.00017488988654901867, + "loss": 0.5578182220458985, + "mean_token_accuracy": 0.830260409921559, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5831904768943786, + "epoch": 1.2259054634745243, + "grad_norm": 0.4383259117603302, + "learning_rate": 0.00017467872603358825, + "loss": 0.5399176788330078, + "mean_token_accuracy": 0.8351450267434121, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5885515750944614, + "epoch": 1.3486801718845918, + "grad_norm": 0.5418892502784729, + "learning_rate": 0.00017430800739444695, + "loss": 0.5402648544311524, + "mean_token_accuracy": 0.8344706323742866, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.603163862079382, + "epoch": 1.4714548802946594, + "grad_norm": 0.5235878825187683, + "learning_rate": 0.0001737784089172037, + "loss": 0.5534031677246094, + "mean_token_accuracy": 0.8304652521014213, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5745155589282512, + "epoch": 1.5942295887047269, + "grad_norm": 0.46109989285469055, + "learning_rate": 0.00017309089958207074, + "loss": 0.5251047134399414, + "mean_token_accuracy": 0.8380702176690101, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5756107035279274, + "epoch": 1.7170042971147943, + "grad_norm": 0.38209712505340576, + "learning_rate": 0.00017224673729096835, + "loss": 0.5253059387207031, + "mean_token_accuracy": 0.8379077833890914, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5695902299880982, + "epoch": 1.839779005524862, + "grad_norm": 0.4063270688056946, + "learning_rate": 0.00017124746656600374, + "loss": 0.5199185180664062, + "mean_token_accuracy": 0.8399301299452782, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5635769453644752, + "epoch": 1.9625537139349294, + "grad_norm": 0.4165436029434204, + "learning_rate": 0.00017009491572353503, + "loss": 0.5156089019775391, + "mean_token_accuracy": 0.8404546654224396, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5587481154714312, + "eval_loss": 0.5402795672416687, + "eval_mean_token_accuracy": 0.8386626076698304, + "eval_num_tokens": 2203536.0, + "eval_runtime": 105.9883, + "eval_samples_per_second": 13.162, + "eval_steps_per_second": 1.651, + "step": 816 + }, + { + "entropy": 0.5002549305785126, + "epoch": 2.0834868017188457, + "grad_norm": 0.4061429500579834, + "learning_rate": 0.00016879119352899107, + "loss": 0.4499735641479492, + "mean_token_accuracy": 0.8566478448470837, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.4894501394033432, + "epoch": 2.2062615101289134, + "grad_norm": 0.530322790145874, + "learning_rate": 0.00016733868533856737, + "loss": 0.4376458740234375, + "mean_token_accuracy": 0.859256779551506, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.48188683822751044, + "epoch": 2.329036218538981, + "grad_norm": 0.4338841438293457, + "learning_rate": 0.0001657400487348575, + "loss": 0.435361328125, + "mean_token_accuracy": 0.8610378202795982, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.4992289599776268, + "epoch": 2.4518109269490487, + "grad_norm": 0.44097110629081726, + "learning_rate": 0.00016399820866440517, + "loss": 0.4474017333984375, + "mean_token_accuracy": 0.8565557089447975, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.49334951683878897, + "epoch": 2.574585635359116, + "grad_norm": 0.48084133863449097, + "learning_rate": 0.00016211635208607405, + "loss": 0.440751953125, + "mean_token_accuracy": 0.8580261027812958, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.4898635870218277, + "epoch": 2.6973603437691835, + "grad_norm": 0.40215447545051575, + "learning_rate": 0.00016009792214002617, + "loss": 0.44588775634765626, + "mean_token_accuracy": 0.8572473660111427, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.4932038363814354, + "epoch": 2.820135052179251, + "grad_norm": 0.4465612769126892, + "learning_rate": 0.00015794661184797816, + "loss": 0.4444055938720703, + "mean_token_accuracy": 0.8583759877085686, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.4850301502645016, + "epoch": 2.942909760589319, + "grad_norm": 0.43597331643104553, + "learning_rate": 0.0001556663573562617, + "loss": 0.4382463073730469, + "mean_token_accuracy": 0.8603282216191291, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.4902002043383462, + "eval_loss": 0.5276822447776794, + "eval_mean_token_accuracy": 0.8454030500139509, + "eval_num_tokens": 3305304.0, + "eval_runtime": 106.004, + "eval_samples_per_second": 13.16, + "eval_steps_per_second": 1.651, + "step": 1224 + }, + { + "entropy": 0.4309167733349776, + "epoch": 3.063842848373235, + "grad_norm": 0.4111804664134979, + "learning_rate": 0.00015326133073405068, + "loss": 0.3837854766845703, + "mean_token_accuracy": 0.8741549949355537, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.38693042084574697, + "epoch": 3.1866175567833026, + "grad_norm": 0.46740710735321045, + "learning_rate": 0.00015073593233993186, + "loss": 0.335601806640625, + "mean_token_accuracy": 0.8861287096142769, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.39316988229751587, + "epoch": 3.3093922651933703, + "grad_norm": 0.5308303236961365, + "learning_rate": 0.00014809478277078602, + "loss": 0.34588619232177736, + "mean_token_accuracy": 0.8837816697359085, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.39958367675542833, + "epoch": 3.4321669736034375, + "grad_norm": 0.5955453515052795, + "learning_rate": 0.00014534271440770966, + "loss": 0.34634483337402344, + "mean_token_accuracy": 0.8832473835349083, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.4100498601794243, + "epoch": 3.554941682013505, + "grad_norm": 0.4593118727207184, + "learning_rate": 0.0001424847625744456, + "loss": 0.3560930633544922, + "mean_token_accuracy": 0.8804642978310585, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.39962531872093676, + "epoch": 3.677716390423573, + "grad_norm": 0.4584615230560303, + "learning_rate": 0.00013952615632449963, + "loss": 0.34978450775146486, + "mean_token_accuracy": 0.8825609818100929, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.39326138779520986, + "epoch": 3.80049109883364, + "grad_norm": 0.5227728486061096, + "learning_rate": 0.00013647230887379902, + "loss": 0.3470985794067383, + "mean_token_accuracy": 0.8833131140470505, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.399793985709548, + "epoch": 3.9232658072437077, + "grad_norm": 0.4263511002063751, + "learning_rate": 0.00013332880769639856, + "loss": 0.3550622177124023, + "mean_token_accuracy": 0.8825028780102729, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.41604454040527344, + "eval_loss": 0.5501811504364014, + "eval_mean_token_accuracy": 0.8466314860752651, + "eval_num_tokens": 4407072.0, + "eval_runtime": 105.9531, + "eval_samples_per_second": 13.166, + "eval_steps_per_second": 1.652, + "step": 1632 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.365209251156378e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbc706f6b7eb2cc5d17ef1de0d7a89ab44c9683 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.08935122001117755, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/tokenizer_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f9fbf8ec3cdac1fcec4aad9042eba908cf582f60 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/trainer_state.json @@ -0,0 +1,489 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2040, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4665306943655014, + "epoch": 0.12277470841006753, + "grad_norm": 0.8344842791557312, + "learning_rate": 2.1010394822962346e-05, + "loss": 1.379196014404297, + "mean_token_accuracy": 0.6848452094197274, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.7834547007083893, + "epoch": 0.24554941682013506, + "grad_norm": 1.1248278617858887, + "learning_rate": 4.2449573213740254e-05, + "loss": 0.7345225524902343, + "mean_token_accuracy": 0.7882961884140969, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7194462446868419, + "epoch": 0.3683241252302026, + "grad_norm": 0.6498261094093323, + "learning_rate": 6.388875160451815e-05, + "loss": 0.664264907836914, + "mean_token_accuracy": 0.8042438223958015, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6771699503064156, + "epoch": 0.4910988336402701, + "grad_norm": 0.6653317809104919, + "learning_rate": 8.532792999529606e-05, + "loss": 0.6322076797485352, + "mean_token_accuracy": 0.8130025643110276, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6632554066181183, + "epoch": 0.6138735420503376, + "grad_norm": 0.44293469190597534, + "learning_rate": 0.00010676710838607398, + "loss": 0.617110595703125, + "mean_token_accuracy": 0.8168379080295562, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6508415986597538, + "epoch": 0.7366482504604052, + "grad_norm": 0.49295490980148315, + "learning_rate": 0.00012820628677685187, + "loss": 0.6023779296875, + "mean_token_accuracy": 0.8187075120210647, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6380941693484783, + "epoch": 0.8594229588704727, + "grad_norm": 0.5855488181114197, + "learning_rate": 0.00014964546516762976, + "loss": 0.5912636947631836, + "mean_token_accuracy": 0.8227647486329078, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.6346297286450863, + "epoch": 0.9821976672805403, + "grad_norm": 0.3817698657512665, + "learning_rate": 0.00017108464355840766, + "loss": 0.5902091598510742, + "mean_token_accuracy": 0.8231476756930352, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6600051839011056, + "eval_loss": 0.5857571363449097, + "eval_mean_token_accuracy": 0.8248803649629866, + "eval_num_tokens": 1101768.0, + "eval_runtime": 106.3263, + "eval_samples_per_second": 13.12, + "eval_steps_per_second": 1.646, + "step": 408 + }, + { + "entropy": 0.5965584144374441, + "epoch": 1.1031307550644567, + "grad_norm": 0.5178472399711609, + "learning_rate": 0.00017488988654901867, + "loss": 0.5578182220458985, + "mean_token_accuracy": 0.830260409921559, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5831904768943786, + "epoch": 1.2259054634745243, + "grad_norm": 0.4383259117603302, + "learning_rate": 0.00017467872603358825, + "loss": 0.5399176788330078, + "mean_token_accuracy": 0.8351450267434121, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5885515750944614, + "epoch": 1.3486801718845918, + "grad_norm": 0.5418892502784729, + "learning_rate": 0.00017430800739444695, + "loss": 0.5402648544311524, + "mean_token_accuracy": 0.8344706323742866, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.603163862079382, + "epoch": 1.4714548802946594, + "grad_norm": 0.5235878825187683, + "learning_rate": 0.0001737784089172037, + "loss": 0.5534031677246094, + "mean_token_accuracy": 0.8304652521014213, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5745155589282512, + "epoch": 1.5942295887047269, + "grad_norm": 0.46109989285469055, + "learning_rate": 0.00017309089958207074, + "loss": 0.5251047134399414, + "mean_token_accuracy": 0.8380702176690101, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5756107035279274, + "epoch": 1.7170042971147943, + "grad_norm": 0.38209712505340576, + "learning_rate": 0.00017224673729096835, + "loss": 0.5253059387207031, + "mean_token_accuracy": 0.8379077833890914, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5695902299880982, + "epoch": 1.839779005524862, + "grad_norm": 0.4063270688056946, + "learning_rate": 0.00017124746656600374, + "loss": 0.5199185180664062, + "mean_token_accuracy": 0.8399301299452782, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5635769453644752, + "epoch": 1.9625537139349294, + "grad_norm": 0.4165436029434204, + "learning_rate": 0.00017009491572353503, + "loss": 0.5156089019775391, + "mean_token_accuracy": 0.8404546654224396, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5587481154714312, + "eval_loss": 0.5402795672416687, + "eval_mean_token_accuracy": 0.8386626076698304, + "eval_num_tokens": 2203536.0, + "eval_runtime": 105.9883, + "eval_samples_per_second": 13.162, + "eval_steps_per_second": 1.651, + "step": 816 + }, + { + "entropy": 0.5002549305785126, + "epoch": 2.0834868017188457, + "grad_norm": 0.4061429500579834, + "learning_rate": 0.00016879119352899107, + "loss": 0.4499735641479492, + "mean_token_accuracy": 0.8566478448470837, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.4894501394033432, + "epoch": 2.2062615101289134, + "grad_norm": 0.530322790145874, + "learning_rate": 0.00016733868533856737, + "loss": 0.4376458740234375, + "mean_token_accuracy": 0.859256779551506, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.48188683822751044, + "epoch": 2.329036218538981, + "grad_norm": 0.4338841438293457, + "learning_rate": 0.0001657400487348575, + "loss": 0.435361328125, + "mean_token_accuracy": 0.8610378202795982, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.4992289599776268, + "epoch": 2.4518109269490487, + "grad_norm": 0.44097110629081726, + "learning_rate": 0.00016399820866440517, + "loss": 0.4474017333984375, + "mean_token_accuracy": 0.8565557089447975, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.49334951683878897, + "epoch": 2.574585635359116, + "grad_norm": 0.48084133863449097, + "learning_rate": 0.00016211635208607405, + "loss": 0.440751953125, + "mean_token_accuracy": 0.8580261027812958, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.4898635870218277, + "epoch": 2.6973603437691835, + "grad_norm": 0.40215447545051575, + "learning_rate": 0.00016009792214002617, + "loss": 0.44588775634765626, + "mean_token_accuracy": 0.8572473660111427, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.4932038363814354, + "epoch": 2.820135052179251, + "grad_norm": 0.4465612769126892, + "learning_rate": 0.00015794661184797816, + "loss": 0.4444055938720703, + "mean_token_accuracy": 0.8583759877085686, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.4850301502645016, + "epoch": 2.942909760589319, + "grad_norm": 0.43597331643104553, + "learning_rate": 0.0001556663573562617, + "loss": 0.4382463073730469, + "mean_token_accuracy": 0.8603282216191291, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.4902002043383462, + "eval_loss": 0.5276822447776794, + "eval_mean_token_accuracy": 0.8454030500139509, + "eval_num_tokens": 3305304.0, + "eval_runtime": 106.004, + "eval_samples_per_second": 13.16, + "eval_steps_per_second": 1.651, + "step": 1224 + }, + { + "entropy": 0.4309167733349776, + "epoch": 3.063842848373235, + "grad_norm": 0.4111804664134979, + "learning_rate": 0.00015326133073405068, + "loss": 0.3837854766845703, + "mean_token_accuracy": 0.8741549949355537, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.38693042084574697, + "epoch": 3.1866175567833026, + "grad_norm": 0.46740710735321045, + "learning_rate": 0.00015073593233993186, + "loss": 0.335601806640625, + "mean_token_accuracy": 0.8861287096142769, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.39316988229751587, + "epoch": 3.3093922651933703, + "grad_norm": 0.5308303236961365, + "learning_rate": 0.00014809478277078602, + "loss": 0.34588619232177736, + "mean_token_accuracy": 0.8837816697359085, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.39958367675542833, + "epoch": 3.4321669736034375, + "grad_norm": 0.5955453515052795, + "learning_rate": 0.00014534271440770966, + "loss": 0.34634483337402344, + "mean_token_accuracy": 0.8832473835349083, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.4100498601794243, + "epoch": 3.554941682013505, + "grad_norm": 0.4593118727207184, + "learning_rate": 0.0001424847625744456, + "loss": 0.3560930633544922, + "mean_token_accuracy": 0.8804642978310585, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.39962531872093676, + "epoch": 3.677716390423573, + "grad_norm": 0.4584615230560303, + "learning_rate": 0.00013952615632449963, + "loss": 0.34978450775146486, + "mean_token_accuracy": 0.8825609818100929, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.39326138779520986, + "epoch": 3.80049109883364, + "grad_norm": 0.5227728486061096, + "learning_rate": 0.00013647230887379902, + "loss": 0.3470985794067383, + "mean_token_accuracy": 0.8833131140470505, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.399793985709548, + "epoch": 3.9232658072437077, + "grad_norm": 0.4263511002063751, + "learning_rate": 0.00013332880769639856, + "loss": 0.3550622177124023, + "mean_token_accuracy": 0.8825028780102729, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.41604454040527344, + "eval_loss": 0.5501811504364014, + "eval_mean_token_accuracy": 0.8466314860752651, + "eval_num_tokens": 4407072.0, + "eval_runtime": 105.9531, + "eval_samples_per_second": 13.166, + "eval_steps_per_second": 1.652, + "step": 1632 + }, + { + "entropy": 0.3492097145107192, + "epoch": 4.044198895027624, + "grad_norm": 0.41719651222229004, + "learning_rate": 0.00013010140430135466, + "loss": 0.30388387680053713, + "mean_token_accuracy": 0.8972433387930623, + "num_tokens": 4455549.0, + "step": 1650 + }, + { + "entropy": 0.2980811195075512, + "epoch": 4.166973603437691, + "grad_norm": 0.694450318813324, + "learning_rate": 0.00012679600370947314, + "loss": 0.24677230834960937, + "mean_token_accuracy": 0.9134943392872811, + "num_tokens": 4584218.0, + "step": 1700 + }, + { + "entropy": 0.2911450309306383, + "epoch": 4.2897483118477595, + "grad_norm": 0.4893203675746918, + "learning_rate": 0.00012341865364918372, + "loss": 0.24411561965942383, + "mean_token_accuracy": 0.9145414170622825, + "num_tokens": 4724425.0, + "step": 1750 + }, + { + "entropy": 0.2982170185446739, + "epoch": 4.412523020257827, + "grad_norm": 0.5446121096611023, + "learning_rate": 0.00011997553349130966, + "loss": 0.24687444686889648, + "mean_token_accuracy": 0.9126561579108238, + "num_tokens": 4861552.0, + "step": 1800 + }, + { + "entropy": 0.30126186162233354, + "epoch": 4.535297728667894, + "grad_norm": 0.5465192198753357, + "learning_rate": 0.00011647294294297787, + "loss": 0.2527424621582031, + "mean_token_accuracy": 0.9110122618079185, + "num_tokens": 4993610.0, + "step": 1850 + }, + { + "entropy": 0.2993039993941784, + "epoch": 4.658072437077962, + "grad_norm": 0.6376634836196899, + "learning_rate": 0.00011291729052135527, + "loss": 0.2507955169677734, + "mean_token_accuracy": 0.9124656105041504, + "num_tokens": 5126291.0, + "step": 1900 + }, + { + "entropy": 0.29630616143345834, + "epoch": 4.780847145488029, + "grad_norm": 0.5191586017608643, + "learning_rate": 0.00010931508182830125, + "loss": 0.2503428077697754, + "mean_token_accuracy": 0.9127612486481667, + "num_tokens": 5266076.0, + "step": 1950 + }, + { + "entropy": 0.29707231901586056, + "epoch": 4.903621853898097, + "grad_norm": 0.42329415678977966, + "learning_rate": 0.00010567290764738874, + "loss": 0.24930551528930664, + "mean_token_accuracy": 0.911863605082035, + "num_tokens": 5403761.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.34850709250995093, + "eval_loss": 0.6053183078765869, + "eval_mean_token_accuracy": 0.8461176470347813, + "eval_num_tokens": 5508840.0, + "eval_runtime": 105.9653, + "eval_samples_per_second": 13.165, + "eval_steps_per_second": 1.651, + "step": 2040 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.203960855839949e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/README.md b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/adapter_config.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbc706f6b7eb2cc5d17ef1de0d7a89ab44c9683 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.08935122001117755, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "q_proj", + "up_proj", + "v_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/chat_template.jinja b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/trainer_state.json b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..70f3fdf4742972649312ad0df7029530a98050c0 --- /dev/null +++ b/random_original_Estonian/Qwen3-14B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/trainer_state.json @@ -0,0 +1,580 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2448, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4665306943655014, + "epoch": 0.12277470841006753, + "grad_norm": 0.8344842791557312, + "learning_rate": 2.1010394822962346e-05, + "loss": 1.379196014404297, + "mean_token_accuracy": 0.6848452094197274, + "num_tokens": 137019.0, + "step": 50 + }, + { + "entropy": 0.7834547007083893, + "epoch": 0.24554941682013506, + "grad_norm": 1.1248278617858887, + "learning_rate": 4.2449573213740254e-05, + "loss": 0.7345225524902343, + "mean_token_accuracy": 0.7882961884140969, + "num_tokens": 267448.0, + "step": 100 + }, + { + "entropy": 0.7194462446868419, + "epoch": 0.3683241252302026, + "grad_norm": 0.6498261094093323, + "learning_rate": 6.388875160451815e-05, + "loss": 0.664264907836914, + "mean_token_accuracy": 0.8042438223958015, + "num_tokens": 409071.0, + "step": 150 + }, + { + "entropy": 0.6771699503064156, + "epoch": 0.4910988336402701, + "grad_norm": 0.6653317809104919, + "learning_rate": 8.532792999529606e-05, + "loss": 0.6322076797485352, + "mean_token_accuracy": 0.8130025643110276, + "num_tokens": 542207.0, + "step": 200 + }, + { + "entropy": 0.6632554066181183, + "epoch": 0.6138735420503376, + "grad_norm": 0.44293469190597534, + "learning_rate": 0.00010676710838607398, + "loss": 0.617110595703125, + "mean_token_accuracy": 0.8168379080295562, + "num_tokens": 679619.0, + "step": 250 + }, + { + "entropy": 0.6508415986597538, + "epoch": 0.7366482504604052, + "grad_norm": 0.49295490980148315, + "learning_rate": 0.00012820628677685187, + "loss": 0.6023779296875, + "mean_token_accuracy": 0.8187075120210647, + "num_tokens": 815397.0, + "step": 300 + }, + { + "entropy": 0.6380941693484783, + "epoch": 0.8594229588704727, + "grad_norm": 0.5855488181114197, + "learning_rate": 0.00014964546516762976, + "loss": 0.5912636947631836, + "mean_token_accuracy": 0.8227647486329078, + "num_tokens": 949315.0, + "step": 350 + }, + { + "entropy": 0.6346297286450863, + "epoch": 0.9821976672805403, + "grad_norm": 0.3817698657512665, + "learning_rate": 0.00017108464355840766, + "loss": 0.5902091598510742, + "mean_token_accuracy": 0.8231476756930352, + "num_tokens": 1082903.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6600051839011056, + "eval_loss": 0.5857571363449097, + "eval_mean_token_accuracy": 0.8248803649629866, + "eval_num_tokens": 1101768.0, + "eval_runtime": 106.3263, + "eval_samples_per_second": 13.12, + "eval_steps_per_second": 1.646, + "step": 408 + }, + { + "entropy": 0.5965584144374441, + "epoch": 1.1031307550644567, + "grad_norm": 0.5178472399711609, + "learning_rate": 0.00017488988654901867, + "loss": 0.5578182220458985, + "mean_token_accuracy": 0.830260409921559, + "num_tokens": 1223073.0, + "step": 450 + }, + { + "entropy": 0.5831904768943786, + "epoch": 1.2259054634745243, + "grad_norm": 0.4383259117603302, + "learning_rate": 0.00017467872603358825, + "loss": 0.5399176788330078, + "mean_token_accuracy": 0.8351450267434121, + "num_tokens": 1360978.0, + "step": 500 + }, + { + "entropy": 0.5885515750944614, + "epoch": 1.3486801718845918, + "grad_norm": 0.5418892502784729, + "learning_rate": 0.00017430800739444695, + "loss": 0.5402648544311524, + "mean_token_accuracy": 0.8344706323742866, + "num_tokens": 1492506.0, + "step": 550 + }, + { + "entropy": 0.603163862079382, + "epoch": 1.4714548802946594, + "grad_norm": 0.5235878825187683, + "learning_rate": 0.0001737784089172037, + "loss": 0.5534031677246094, + "mean_token_accuracy": 0.8304652521014213, + "num_tokens": 1622275.0, + "step": 600 + }, + { + "entropy": 0.5745155589282512, + "epoch": 1.5942295887047269, + "grad_norm": 0.46109989285469055, + "learning_rate": 0.00017309089958207074, + "loss": 0.5251047134399414, + "mean_token_accuracy": 0.8380702176690101, + "num_tokens": 1756450.0, + "step": 650 + }, + { + "entropy": 0.5756107035279274, + "epoch": 1.7170042971147943, + "grad_norm": 0.38209712505340576, + "learning_rate": 0.00017224673729096835, + "loss": 0.5253059387207031, + "mean_token_accuracy": 0.8379077833890914, + "num_tokens": 1890634.0, + "step": 700 + }, + { + "entropy": 0.5695902299880982, + "epoch": 1.839779005524862, + "grad_norm": 0.4063270688056946, + "learning_rate": 0.00017124746656600374, + "loss": 0.5199185180664062, + "mean_token_accuracy": 0.8399301299452782, + "num_tokens": 2028046.0, + "step": 750 + }, + { + "entropy": 0.5635769453644752, + "epoch": 1.9625537139349294, + "grad_norm": 0.4165436029434204, + "learning_rate": 0.00017009491572353503, + "loss": 0.5156089019775391, + "mean_token_accuracy": 0.8404546654224396, + "num_tokens": 2163703.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5587481154714312, + "eval_loss": 0.5402795672416687, + "eval_mean_token_accuracy": 0.8386626076698304, + "eval_num_tokens": 2203536.0, + "eval_runtime": 105.9883, + "eval_samples_per_second": 13.162, + "eval_steps_per_second": 1.651, + "step": 816 + }, + { + "entropy": 0.5002549305785126, + "epoch": 2.0834868017188457, + "grad_norm": 0.4061429500579834, + "learning_rate": 0.00016879119352899107, + "loss": 0.4499735641479492, + "mean_token_accuracy": 0.8566478448470837, + "num_tokens": 2298231.0, + "step": 850 + }, + { + "entropy": 0.4894501394033432, + "epoch": 2.2062615101289134, + "grad_norm": 0.530322790145874, + "learning_rate": 0.00016733868533856737, + "loss": 0.4376458740234375, + "mean_token_accuracy": 0.859256779551506, + "num_tokens": 2427608.0, + "step": 900 + }, + { + "entropy": 0.48188683822751044, + "epoch": 2.329036218538981, + "grad_norm": 0.4338841438293457, + "learning_rate": 0.0001657400487348575, + "loss": 0.435361328125, + "mean_token_accuracy": 0.8610378202795982, + "num_tokens": 2565982.0, + "step": 950 + }, + { + "entropy": 0.4992289599776268, + "epoch": 2.4518109269490487, + "grad_norm": 0.44097110629081726, + "learning_rate": 0.00016399820866440517, + "loss": 0.4474017333984375, + "mean_token_accuracy": 0.8565557089447975, + "num_tokens": 2695545.0, + "step": 1000 + }, + { + "entropy": 0.49334951683878897, + "epoch": 2.574585635359116, + "grad_norm": 0.48084133863449097, + "learning_rate": 0.00016211635208607405, + "loss": 0.440751953125, + "mean_token_accuracy": 0.8580261027812958, + "num_tokens": 2829781.0, + "step": 1050 + }, + { + "entropy": 0.4898635870218277, + "epoch": 2.6973603437691835, + "grad_norm": 0.40215447545051575, + "learning_rate": 0.00016009792214002617, + "loss": 0.44588775634765626, + "mean_token_accuracy": 0.8572473660111427, + "num_tokens": 2970162.0, + "step": 1100 + }, + { + "entropy": 0.4932038363814354, + "epoch": 2.820135052179251, + "grad_norm": 0.4465612769126892, + "learning_rate": 0.00015794661184797816, + "loss": 0.4444055938720703, + "mean_token_accuracy": 0.8583759877085686, + "num_tokens": 3103775.0, + "step": 1150 + }, + { + "entropy": 0.4850301502645016, + "epoch": 2.942909760589319, + "grad_norm": 0.43597331643104553, + "learning_rate": 0.0001556663573562617, + "loss": 0.4382463073730469, + "mean_token_accuracy": 0.8603282216191291, + "num_tokens": 3241634.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.4902002043383462, + "eval_loss": 0.5276822447776794, + "eval_mean_token_accuracy": 0.8454030500139509, + "eval_num_tokens": 3305304.0, + "eval_runtime": 106.004, + "eval_samples_per_second": 13.16, + "eval_steps_per_second": 1.651, + "step": 1224 + }, + { + "entropy": 0.4309167733349776, + "epoch": 3.063842848373235, + "grad_norm": 0.4111804664134979, + "learning_rate": 0.00015326133073405068, + "loss": 0.3837854766845703, + "mean_token_accuracy": 0.8741549949355537, + "num_tokens": 3378837.0, + "step": 1250 + }, + { + "entropy": 0.38693042084574697, + "epoch": 3.1866175567833026, + "grad_norm": 0.46740710735321045, + "learning_rate": 0.00015073593233993186, + "loss": 0.335601806640625, + "mean_token_accuracy": 0.8861287096142769, + "num_tokens": 3510318.0, + "step": 1300 + }, + { + "entropy": 0.39316988229751587, + "epoch": 3.3093922651933703, + "grad_norm": 0.5308303236961365, + "learning_rate": 0.00014809478277078602, + "loss": 0.34588619232177736, + "mean_token_accuracy": 0.8837816697359085, + "num_tokens": 3647579.0, + "step": 1350 + }, + { + "entropy": 0.39958367675542833, + "epoch": 3.4321669736034375, + "grad_norm": 0.5955453515052795, + "learning_rate": 0.00014534271440770966, + "loss": 0.34634483337402344, + "mean_token_accuracy": 0.8832473835349083, + "num_tokens": 3779946.0, + "step": 1400 + }, + { + "entropy": 0.4100498601794243, + "epoch": 3.554941682013505, + "grad_norm": 0.4593118727207184, + "learning_rate": 0.0001424847625744456, + "loss": 0.3560930633544922, + "mean_token_accuracy": 0.8804642978310585, + "num_tokens": 3908873.0, + "step": 1450 + }, + { + "entropy": 0.39962531872093676, + "epoch": 3.677716390423573, + "grad_norm": 0.4584615230560303, + "learning_rate": 0.00013952615632449963, + "loss": 0.34978450775146486, + "mean_token_accuracy": 0.8825609818100929, + "num_tokens": 4046620.0, + "step": 1500 + }, + { + "entropy": 0.39326138779520986, + "epoch": 3.80049109883364, + "grad_norm": 0.5227728486061096, + "learning_rate": 0.00013647230887379902, + "loss": 0.3470985794067383, + "mean_token_accuracy": 0.8833131140470505, + "num_tokens": 4184429.0, + "step": 1550 + }, + { + "entropy": 0.399793985709548, + "epoch": 3.9232658072437077, + "grad_norm": 0.4263511002063751, + "learning_rate": 0.00013332880769639856, + "loss": 0.3550622177124023, + "mean_token_accuracy": 0.8825028780102729, + "num_tokens": 4318596.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.41604454040527344, + "eval_loss": 0.5501811504364014, + "eval_mean_token_accuracy": 0.8466314860752651, + "eval_num_tokens": 4407072.0, + "eval_runtime": 105.9531, + "eval_samples_per_second": 13.166, + "eval_steps_per_second": 1.652, + "step": 1632 + }, + { + "entropy": 0.3492097145107192, + "epoch": 4.044198895027624, + "grad_norm": 0.41719651222229004, + "learning_rate": 0.00013010140430135466, + "loss": 0.30388387680053713, + "mean_token_accuracy": 0.8972433387930623, + "num_tokens": 4455549.0, + "step": 1650 + }, + { + "entropy": 0.2980811195075512, + "epoch": 4.166973603437691, + "grad_norm": 0.694450318813324, + "learning_rate": 0.00012679600370947314, + "loss": 0.24677230834960937, + "mean_token_accuracy": 0.9134943392872811, + "num_tokens": 4584218.0, + "step": 1700 + }, + { + "entropy": 0.2911450309306383, + "epoch": 4.2897483118477595, + "grad_norm": 0.4893203675746918, + "learning_rate": 0.00012341865364918372, + "loss": 0.24411561965942383, + "mean_token_accuracy": 0.9145414170622825, + "num_tokens": 4724425.0, + "step": 1750 + }, + { + "entropy": 0.2982170185446739, + "epoch": 4.412523020257827, + "grad_norm": 0.5446121096611023, + "learning_rate": 0.00011997553349130966, + "loss": 0.24687444686889648, + "mean_token_accuracy": 0.9126561579108238, + "num_tokens": 4861552.0, + "step": 1800 + }, + { + "entropy": 0.30126186162233354, + "epoch": 4.535297728667894, + "grad_norm": 0.5465192198753357, + "learning_rate": 0.00011647294294297787, + "loss": 0.2527424621582031, + "mean_token_accuracy": 0.9110122618079185, + "num_tokens": 4993610.0, + "step": 1850 + }, + { + "entropy": 0.2993039993941784, + "epoch": 4.658072437077962, + "grad_norm": 0.6376634836196899, + "learning_rate": 0.00011291729052135527, + "loss": 0.2507955169677734, + "mean_token_accuracy": 0.9124656105041504, + "num_tokens": 5126291.0, + "step": 1900 + }, + { + "entropy": 0.29630616143345834, + "epoch": 4.780847145488029, + "grad_norm": 0.5191586017608643, + "learning_rate": 0.00010931508182830125, + "loss": 0.2503428077697754, + "mean_token_accuracy": 0.9127612486481667, + "num_tokens": 5266076.0, + "step": 1950 + }, + { + "entropy": 0.29707231901586056, + "epoch": 4.903621853898097, + "grad_norm": 0.42329415678977966, + "learning_rate": 0.00010567290764738874, + "loss": 0.24930551528930664, + "mean_token_accuracy": 0.911863605082035, + "num_tokens": 5403761.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.34850709250995093, + "eval_loss": 0.6053183078765869, + "eval_mean_token_accuracy": 0.8461176470347813, + "eval_num_tokens": 5508840.0, + "eval_runtime": 105.9653, + "eval_samples_per_second": 13.165, + "eval_steps_per_second": 1.651, + "step": 2040 + }, + { + "entropy": 0.28518860919523964, + "epoch": 5.024554941682013, + "grad_norm": 0.6316563487052917, + "learning_rate": 0.00010199743188507247, + "loss": 0.23341741561889648, + "mean_token_accuracy": 0.9183166952907736, + "num_tokens": 5535743.0, + "step": 2050 + }, + { + "entropy": 0.20185263987630606, + "epoch": 5.147329650092081, + "grad_norm": 0.6542733311653137, + "learning_rate": 9.829537937806829e-05, + "loss": 0.1496055793762207, + "mean_token_accuracy": 0.9459337556362152, + "num_tokens": 5675294.0, + "step": 2100 + }, + { + "entropy": 0.2101737355440855, + "epoch": 5.270104358502149, + "grad_norm": 0.46600785851478577, + "learning_rate": 9.457352358925075e-05, + "loss": 0.15642745018005372, + "mean_token_accuracy": 0.944338481426239, + "num_tokens": 5808279.0, + "step": 2150 + }, + { + "entropy": 0.200700718536973, + "epoch": 5.392879066912216, + "grad_norm": 0.5562126636505127, + "learning_rate": 9.083867421458257e-05, + "loss": 0.15122636795043945, + "mean_token_accuracy": 0.9447117391228675, + "num_tokens": 5948774.0, + "step": 2200 + }, + { + "entropy": 0.2139152016490698, + "epoch": 5.515653775322283, + "grad_norm": 0.5136730670928955, + "learning_rate": 8.709766472374944e-05, + "loss": 0.16092844009399415, + "mean_token_accuracy": 0.9423242849111557, + "num_tokens": 6081211.0, + "step": 2250 + }, + { + "entropy": 0.20662450045347214, + "epoch": 5.638428483732351, + "grad_norm": 0.6007277965545654, + "learning_rate": 8.335733985729791e-05, + "loss": 0.15495609283447265, + "mean_token_accuracy": 0.9431806969642639, + "num_tokens": 6213793.0, + "step": 2300 + }, + { + "entropy": 0.20562831707298757, + "epoch": 5.7612031921424185, + "grad_norm": 0.5216547846794128, + "learning_rate": 7.962454310315095e-05, + "loss": 0.1568380641937256, + "mean_token_accuracy": 0.9425676354765892, + "num_tokens": 6350388.0, + "step": 2350 + }, + { + "entropy": 0.20440548222512006, + "epoch": 5.883977900552486, + "grad_norm": 0.5724782943725586, + "learning_rate": 7.590610417541581e-05, + "loss": 0.15627790451049806, + "mean_token_accuracy": 0.9437827947735786, + "num_tokens": 6485960.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.2856926244497299, + "eval_loss": 0.7071177959442139, + "eval_mean_token_accuracy": 0.8432521397726876, + "eval_num_tokens": 6610608.0, + "eval_runtime": 105.9542, + "eval_samples_per_second": 13.166, + "eval_steps_per_second": 1.652, + "step": 2448 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.1042614122067046e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dfa6854aee8f17d1b6fa21e4e34db35f4ef3a40c --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.005046715367502997, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..eff60d6ea40c592dca72cde765d99f429320bbb6 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1224/trainer_state.json @@ -0,0 +1,340 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1224, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8492428332567215, + "epoch": 0.12277470841006753, + "grad_norm": 0.5106982588768005, + "learning_rate": 3.492278846049215e-05, + "loss": 1.7872950744628906, + "mean_token_accuracy": 0.6242395298182964, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8940915194153786, + "epoch": 0.24554941682013506, + "grad_norm": 0.6675613522529602, + "learning_rate": 7.055828688956577e-05, + "loss": 0.8397601318359375, + "mean_token_accuracy": 0.7503150171041488, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7644237650930882, + "epoch": 0.3683241252302026, + "grad_norm": 0.43234843015670776, + "learning_rate": 0.00010619378531863939, + "loss": 0.7161208343505859, + "mean_token_accuracy": 0.778390144109726, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7164452224969864, + "epoch": 0.4910988336402701, + "grad_norm": 0.4049525558948517, + "learning_rate": 0.00014182928374771302, + "loss": 0.6739736938476563, + "mean_token_accuracy": 0.7887251850962639, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7021449965238571, + "epoch": 0.6138735420503376, + "grad_norm": 0.3455510139465332, + "learning_rate": 0.00017746478217678664, + "loss": 0.6584578704833984, + "mean_token_accuracy": 0.7932635286450386, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6882825265824795, + "epoch": 0.7366482504604052, + "grad_norm": 0.30983075499534607, + "learning_rate": 0.00021310028060586025, + "loss": 0.6412281799316406, + "mean_token_accuracy": 0.7957331365346909, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6746629500389099, + "epoch": 0.8594229588704727, + "grad_norm": 0.4235651195049286, + "learning_rate": 0.0002487357790349339, + "loss": 0.6273543167114258, + "mean_token_accuracy": 0.8004380378127098, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6693046718835831, + "epoch": 0.9821976672805403, + "grad_norm": 0.2910408079624176, + "learning_rate": 0.0002843712774640075, + "loss": 0.6247041320800781, + "mean_token_accuracy": 0.8027204623818398, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6973312156541007, + "eval_mean_token_accuracy": 0.7841839187485831, + "eval_not_syn_loss": 0.6392670273780823, + "eval_not_syn_runtime": 106.0995, + "eval_not_syn_samples_per_second": 13.148, + "eval_not_syn_steps_per_second": 1.649, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6523867055347987, + "eval_mean_token_accuracy": 0.8224203096117292, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.5988379120826721, + "eval_syn_runtime": 112.4432, + "eval_syn_samples_per_second": 12.406, + "eval_syn_steps_per_second": 1.556, + "step": 408 + }, + { + "entropy": 0.6396199787026129, + "epoch": 1.1031307550644567, + "grad_norm": 0.3523969054222107, + "learning_rate": 0.0002906962274290906, + "loss": 0.5951201248168946, + "mean_token_accuracy": 0.809070566584011, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6200365445017815, + "epoch": 1.2259054634745243, + "grad_norm": 0.28092390298843384, + "learning_rate": 0.00029034524335317383, + "loss": 0.5773784637451171, + "mean_token_accuracy": 0.8133166086673737, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.62518698990345, + "epoch": 1.3486801718845918, + "grad_norm": 0.34311696887016296, + "learning_rate": 0.0002897290470026443, + "loss": 0.5758082962036133, + "mean_token_accuracy": 0.8123552390933036, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6401302371919155, + "epoch": 1.4714548802946594, + "grad_norm": 0.3884620666503906, + "learning_rate": 0.0002888487658015718, + "loss": 0.58863037109375, + "mean_token_accuracy": 0.8095892634987831, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6160605426132679, + "epoch": 1.5942295887047269, + "grad_norm": 0.34537142515182495, + "learning_rate": 0.00028770601035705147, + "loss": 0.5640074539184571, + "mean_token_accuracy": 0.8150847071409225, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6120209716260433, + "epoch": 1.7170042971147943, + "grad_norm": 0.2542697787284851, + "learning_rate": 0.0002863028715123557, + "loss": 0.5619714736938477, + "mean_token_accuracy": 0.8152382817864418, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6054908482730389, + "epoch": 1.839779005524862, + "grad_norm": 0.3275090456008911, + "learning_rate": 0.0002846419165214213, + "loss": 0.5586128234863281, + "mean_token_accuracy": 0.8170404744148254, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6005898140370846, + "epoch": 1.9625537139349294, + "grad_norm": 0.24491196870803833, + "learning_rate": 0.00028272618435167137, + "loss": 0.5549465179443359, + "mean_token_accuracy": 0.8176508370041847, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6112558654376439, + "eval_mean_token_accuracy": 0.793729545388903, + "eval_not_syn_loss": 0.5950002074241638, + "eval_not_syn_runtime": 105.7692, + "eval_not_syn_samples_per_second": 13.189, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5759283176490239, + "eval_mean_token_accuracy": 0.8401244885580881, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5499612092971802, + "eval_syn_runtime": 112.3691, + "eval_syn_samples_per_second": 12.414, + "eval_syn_steps_per_second": 1.557, + "step": 816 + }, + { + "entropy": 0.5536743695965878, + "epoch": 2.0834868017188457, + "grad_norm": 0.25854116678237915, + "learning_rate": 0.00028055918012376665, + "loss": 0.5038502883911132, + "mean_token_accuracy": 0.8288816407247244, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5468840812146664, + "epoch": 2.2062615101289134, + "grad_norm": 0.35776397585868835, + "learning_rate": 0.00027814486869845917, + "loss": 0.4977967071533203, + "mean_token_accuracy": 0.8320337995886803, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5361524756252766, + "epoch": 2.329036218538981, + "grad_norm": 0.2908971905708313, + "learning_rate": 0.0002754876674222821, + "loss": 0.49344730377197266, + "mean_token_accuracy": 0.8353969538211823, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5515676756203175, + "epoch": 2.4518109269490487, + "grad_norm": 0.2485043853521347, + "learning_rate": 0.0002725924380453484, + "loss": 0.501014518737793, + "mean_token_accuracy": 0.8305466166138649, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5483240431547165, + "epoch": 2.574585635359116, + "grad_norm": 0.27612268924713135, + "learning_rate": 0.000269464477826047, + "loss": 0.4939273834228516, + "mean_token_accuracy": 0.8312122821807861, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5382450266182423, + "epoch": 2.6973603437691835, + "grad_norm": 0.23383453488349915, + "learning_rate": 0.0002661095098389097, + "loss": 0.4967027282714844, + "mean_token_accuracy": 0.8317142990231514, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5448310880362988, + "epoch": 2.820135052179251, + "grad_norm": 0.29418694972991943, + "learning_rate": 0.0002625336725033846, + "loss": 0.4974230194091797, + "mean_token_accuracy": 0.8330811312794686, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.538916184157133, + "epoch": 2.942909760589319, + "grad_norm": 0.2986706495285034, + "learning_rate": 0.0002587435083526725, + "loss": 0.49071075439453127, + "mean_token_accuracy": 0.835035215318203, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5504456080709185, + "eval_mean_token_accuracy": 0.8138916601453509, + "eval_not_syn_loss": 0.5711027383804321, + "eval_not_syn_runtime": 105.759, + "eval_not_syn_samples_per_second": 13.19, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5207725673062461, + "eval_mean_token_accuracy": 0.8279450879778181, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.5415107607841492, + "eval_syn_runtime": 112.382, + "eval_syn_samples_per_second": 12.413, + "eval_syn_steps_per_second": 1.557, + "step": 1224 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.5555392143591424e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dfa6854aee8f17d1b6fa21e4e34db35f4ef3a40c --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.005046715367502997, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..147c7f30e7dbd6c998633893dec015052a556723 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-1632/trainer_state.json @@ -0,0 +1,442 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1632, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8492428332567215, + "epoch": 0.12277470841006753, + "grad_norm": 0.5106982588768005, + "learning_rate": 3.492278846049215e-05, + "loss": 1.7872950744628906, + "mean_token_accuracy": 0.6242395298182964, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8940915194153786, + "epoch": 0.24554941682013506, + "grad_norm": 0.6675613522529602, + "learning_rate": 7.055828688956577e-05, + "loss": 0.8397601318359375, + "mean_token_accuracy": 0.7503150171041488, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7644237650930882, + "epoch": 0.3683241252302026, + "grad_norm": 0.43234843015670776, + "learning_rate": 0.00010619378531863939, + "loss": 0.7161208343505859, + "mean_token_accuracy": 0.778390144109726, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7164452224969864, + "epoch": 0.4910988336402701, + "grad_norm": 0.4049525558948517, + "learning_rate": 0.00014182928374771302, + "loss": 0.6739736938476563, + "mean_token_accuracy": 0.7887251850962639, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7021449965238571, + "epoch": 0.6138735420503376, + "grad_norm": 0.3455510139465332, + "learning_rate": 0.00017746478217678664, + "loss": 0.6584578704833984, + "mean_token_accuracy": 0.7932635286450386, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6882825265824795, + "epoch": 0.7366482504604052, + "grad_norm": 0.30983075499534607, + "learning_rate": 0.00021310028060586025, + "loss": 0.6412281799316406, + "mean_token_accuracy": 0.7957331365346909, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6746629500389099, + "epoch": 0.8594229588704727, + "grad_norm": 0.4235651195049286, + "learning_rate": 0.0002487357790349339, + "loss": 0.6273543167114258, + "mean_token_accuracy": 0.8004380378127098, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6693046718835831, + "epoch": 0.9821976672805403, + "grad_norm": 0.2910408079624176, + "learning_rate": 0.0002843712774640075, + "loss": 0.6247041320800781, + "mean_token_accuracy": 0.8027204623818398, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6973312156541007, + "eval_mean_token_accuracy": 0.7841839187485831, + "eval_not_syn_loss": 0.6392670273780823, + "eval_not_syn_runtime": 106.0995, + "eval_not_syn_samples_per_second": 13.148, + "eval_not_syn_steps_per_second": 1.649, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6523867055347987, + "eval_mean_token_accuracy": 0.8224203096117292, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.5988379120826721, + "eval_syn_runtime": 112.4432, + "eval_syn_samples_per_second": 12.406, + "eval_syn_steps_per_second": 1.556, + "step": 408 + }, + { + "entropy": 0.6396199787026129, + "epoch": 1.1031307550644567, + "grad_norm": 0.3523969054222107, + "learning_rate": 0.0002906962274290906, + "loss": 0.5951201248168946, + "mean_token_accuracy": 0.809070566584011, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6200365445017815, + "epoch": 1.2259054634745243, + "grad_norm": 0.28092390298843384, + "learning_rate": 0.00029034524335317383, + "loss": 0.5773784637451171, + "mean_token_accuracy": 0.8133166086673737, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.62518698990345, + "epoch": 1.3486801718845918, + "grad_norm": 0.34311696887016296, + "learning_rate": 0.0002897290470026443, + "loss": 0.5758082962036133, + "mean_token_accuracy": 0.8123552390933036, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6401302371919155, + "epoch": 1.4714548802946594, + "grad_norm": 0.3884620666503906, + "learning_rate": 0.0002888487658015718, + "loss": 0.58863037109375, + "mean_token_accuracy": 0.8095892634987831, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6160605426132679, + "epoch": 1.5942295887047269, + "grad_norm": 0.34537142515182495, + "learning_rate": 0.00028770601035705147, + "loss": 0.5640074539184571, + "mean_token_accuracy": 0.8150847071409225, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6120209716260433, + "epoch": 1.7170042971147943, + "grad_norm": 0.2542697787284851, + "learning_rate": 0.0002863028715123557, + "loss": 0.5619714736938477, + "mean_token_accuracy": 0.8152382817864418, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6054908482730389, + "epoch": 1.839779005524862, + "grad_norm": 0.3275090456008911, + "learning_rate": 0.0002846419165214213, + "loss": 0.5586128234863281, + "mean_token_accuracy": 0.8170404744148254, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6005898140370846, + "epoch": 1.9625537139349294, + "grad_norm": 0.24491196870803833, + "learning_rate": 0.00028272618435167137, + "loss": 0.5549465179443359, + "mean_token_accuracy": 0.8176508370041847, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6112558654376439, + "eval_mean_token_accuracy": 0.793729545388903, + "eval_not_syn_loss": 0.5950002074241638, + "eval_not_syn_runtime": 105.7692, + "eval_not_syn_samples_per_second": 13.189, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5759283176490239, + "eval_mean_token_accuracy": 0.8401244885580881, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5499612092971802, + "eval_syn_runtime": 112.3691, + "eval_syn_samples_per_second": 12.414, + "eval_syn_steps_per_second": 1.557, + "step": 816 + }, + { + "entropy": 0.5536743695965878, + "epoch": 2.0834868017188457, + "grad_norm": 0.25854116678237915, + "learning_rate": 0.00028055918012376665, + "loss": 0.5038502883911132, + "mean_token_accuracy": 0.8288816407247244, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5468840812146664, + "epoch": 2.2062615101289134, + "grad_norm": 0.35776397585868835, + "learning_rate": 0.00027814486869845917, + "loss": 0.4977967071533203, + "mean_token_accuracy": 0.8320337995886803, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5361524756252766, + "epoch": 2.329036218538981, + "grad_norm": 0.2908971905708313, + "learning_rate": 0.0002754876674222821, + "loss": 0.49344730377197266, + "mean_token_accuracy": 0.8353969538211823, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5515676756203175, + "epoch": 2.4518109269490487, + "grad_norm": 0.2485043853521347, + "learning_rate": 0.0002725924380453484, + "loss": 0.501014518737793, + "mean_token_accuracy": 0.8305466166138649, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5483240431547165, + "epoch": 2.574585635359116, + "grad_norm": 0.27612268924713135, + "learning_rate": 0.000269464477826047, + "loss": 0.4939273834228516, + "mean_token_accuracy": 0.8312122821807861, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5382450266182423, + "epoch": 2.6973603437691835, + "grad_norm": 0.23383453488349915, + "learning_rate": 0.0002661095098389097, + "loss": 0.4967027282714844, + "mean_token_accuracy": 0.8317142990231514, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5448310880362988, + "epoch": 2.820135052179251, + "grad_norm": 0.29418694972991943, + "learning_rate": 0.0002625336725033846, + "loss": 0.4974230194091797, + "mean_token_accuracy": 0.8330811312794686, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.538916184157133, + "epoch": 2.942909760589319, + "grad_norm": 0.2986706495285034, + "learning_rate": 0.0002587435083526725, + "loss": 0.49071075439453127, + "mean_token_accuracy": 0.835035215318203, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5504456080709185, + "eval_mean_token_accuracy": 0.8138916601453509, + "eval_not_syn_loss": 0.5711027383804321, + "eval_not_syn_runtime": 105.759, + "eval_not_syn_samples_per_second": 13.19, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5207725673062461, + "eval_mean_token_accuracy": 0.8279450879778181, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.5415107607841492, + "eval_syn_runtime": 112.382, + "eval_syn_samples_per_second": 12.413, + "eval_syn_steps_per_second": 1.557, + "step": 1224 + }, + { + "entropy": 0.500392770404138, + "epoch": 3.063842848373235, + "grad_norm": 0.24652981758117676, + "learning_rate": 0.0002547459520631764, + "loss": 0.4460899353027344, + "mean_token_accuracy": 0.8447940503885298, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.4625867912173271, + "epoch": 3.1866175567833026, + "grad_norm": 0.2843637466430664, + "learning_rate": 0.00025054831776646675, + "loss": 0.41165496826171877, + "mean_token_accuracy": 0.8544793871045112, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4739935018122196, + "epoch": 3.3093922651933703, + "grad_norm": 0.32995307445526123, + "learning_rate": 0.000246158285666975, + "loss": 0.42344165802001954, + "mean_token_accuracy": 0.8512757349014283, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.47488294705748557, + "epoch": 3.4321669736034375, + "grad_norm": 0.3435416519641876, + "learning_rate": 0.00024158388798990277, + "loss": 0.42080265045166015, + "mean_token_accuracy": 0.8512625667452812, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.4870477955043316, + "epoch": 3.554941682013505, + "grad_norm": 0.33815860748291016, + "learning_rate": 0.0002368334942850555, + "loss": 0.43374271392822267, + "mean_token_accuracy": 0.849637107849121, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.4743228654563427, + "epoch": 3.677716390423573, + "grad_norm": 0.2805832326412201, + "learning_rate": 0.0002319157961134899, + "loss": 0.42299102783203124, + "mean_token_accuracy": 0.85123528778553, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.46706863716244695, + "epoch": 3.80049109883364, + "grad_norm": 0.3060542941093445, + "learning_rate": 0.00022683979114499338, + "loss": 0.4191351318359375, + "mean_token_accuracy": 0.8529838898777962, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4743580640852451, + "epoch": 3.9232658072437077, + "grad_norm": 0.2689199447631836, + "learning_rate": 0.00022161476669549157, + "loss": 0.43219676971435544, + "mean_token_accuracy": 0.8495954439043999, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5012165577071054, + "eval_mean_token_accuracy": 0.8084095290728978, + "eval_not_syn_loss": 0.5803263187408447, + "eval_not_syn_runtime": 105.8265, + "eval_not_syn_samples_per_second": 13.182, + "eval_not_syn_steps_per_second": 1.654, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.47334393518311635, + "eval_mean_token_accuracy": 0.8443782693999154, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5437710881233215, + "eval_syn_runtime": 112.3885, + "eval_syn_samples_per_second": 12.412, + "eval_syn_steps_per_second": 1.557, + "step": 1632 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.408322545728922e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dfa6854aee8f17d1b6fa21e4e34db35f4ef3a40c --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.005046715367502997, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b4e9bb8fb5a8b7d4b41d7f69c70b21339bac8af5 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2040/trainer_state.json @@ -0,0 +1,544 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2040, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8492428332567215, + "epoch": 0.12277470841006753, + "grad_norm": 0.5106982588768005, + "learning_rate": 3.492278846049215e-05, + "loss": 1.7872950744628906, + "mean_token_accuracy": 0.6242395298182964, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8940915194153786, + "epoch": 0.24554941682013506, + "grad_norm": 0.6675613522529602, + "learning_rate": 7.055828688956577e-05, + "loss": 0.8397601318359375, + "mean_token_accuracy": 0.7503150171041488, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7644237650930882, + "epoch": 0.3683241252302026, + "grad_norm": 0.43234843015670776, + "learning_rate": 0.00010619378531863939, + "loss": 0.7161208343505859, + "mean_token_accuracy": 0.778390144109726, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7164452224969864, + "epoch": 0.4910988336402701, + "grad_norm": 0.4049525558948517, + "learning_rate": 0.00014182928374771302, + "loss": 0.6739736938476563, + "mean_token_accuracy": 0.7887251850962639, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7021449965238571, + "epoch": 0.6138735420503376, + "grad_norm": 0.3455510139465332, + "learning_rate": 0.00017746478217678664, + "loss": 0.6584578704833984, + "mean_token_accuracy": 0.7932635286450386, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6882825265824795, + "epoch": 0.7366482504604052, + "grad_norm": 0.30983075499534607, + "learning_rate": 0.00021310028060586025, + "loss": 0.6412281799316406, + "mean_token_accuracy": 0.7957331365346909, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6746629500389099, + "epoch": 0.8594229588704727, + "grad_norm": 0.4235651195049286, + "learning_rate": 0.0002487357790349339, + "loss": 0.6273543167114258, + "mean_token_accuracy": 0.8004380378127098, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6693046718835831, + "epoch": 0.9821976672805403, + "grad_norm": 0.2910408079624176, + "learning_rate": 0.0002843712774640075, + "loss": 0.6247041320800781, + "mean_token_accuracy": 0.8027204623818398, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6973312156541007, + "eval_mean_token_accuracy": 0.7841839187485831, + "eval_not_syn_loss": 0.6392670273780823, + "eval_not_syn_runtime": 106.0995, + "eval_not_syn_samples_per_second": 13.148, + "eval_not_syn_steps_per_second": 1.649, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6523867055347987, + "eval_mean_token_accuracy": 0.8224203096117292, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.5988379120826721, + "eval_syn_runtime": 112.4432, + "eval_syn_samples_per_second": 12.406, + "eval_syn_steps_per_second": 1.556, + "step": 408 + }, + { + "entropy": 0.6396199787026129, + "epoch": 1.1031307550644567, + "grad_norm": 0.3523969054222107, + "learning_rate": 0.0002906962274290906, + "loss": 0.5951201248168946, + "mean_token_accuracy": 0.809070566584011, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6200365445017815, + "epoch": 1.2259054634745243, + "grad_norm": 0.28092390298843384, + "learning_rate": 0.00029034524335317383, + "loss": 0.5773784637451171, + "mean_token_accuracy": 0.8133166086673737, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.62518698990345, + "epoch": 1.3486801718845918, + "grad_norm": 0.34311696887016296, + "learning_rate": 0.0002897290470026443, + "loss": 0.5758082962036133, + "mean_token_accuracy": 0.8123552390933036, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6401302371919155, + "epoch": 1.4714548802946594, + "grad_norm": 0.3884620666503906, + "learning_rate": 0.0002888487658015718, + "loss": 0.58863037109375, + "mean_token_accuracy": 0.8095892634987831, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6160605426132679, + "epoch": 1.5942295887047269, + "grad_norm": 0.34537142515182495, + "learning_rate": 0.00028770601035705147, + "loss": 0.5640074539184571, + "mean_token_accuracy": 0.8150847071409225, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6120209716260433, + "epoch": 1.7170042971147943, + "grad_norm": 0.2542697787284851, + "learning_rate": 0.0002863028715123557, + "loss": 0.5619714736938477, + "mean_token_accuracy": 0.8152382817864418, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6054908482730389, + "epoch": 1.839779005524862, + "grad_norm": 0.3275090456008911, + "learning_rate": 0.0002846419165214213, + "loss": 0.5586128234863281, + "mean_token_accuracy": 0.8170404744148254, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6005898140370846, + "epoch": 1.9625537139349294, + "grad_norm": 0.24491196870803833, + "learning_rate": 0.00028272618435167137, + "loss": 0.5549465179443359, + "mean_token_accuracy": 0.8176508370041847, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6112558654376439, + "eval_mean_token_accuracy": 0.793729545388903, + "eval_not_syn_loss": 0.5950002074241638, + "eval_not_syn_runtime": 105.7692, + "eval_not_syn_samples_per_second": 13.189, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5759283176490239, + "eval_mean_token_accuracy": 0.8401244885580881, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5499612092971802, + "eval_syn_runtime": 112.3691, + "eval_syn_samples_per_second": 12.414, + "eval_syn_steps_per_second": 1.557, + "step": 816 + }, + { + "entropy": 0.5536743695965878, + "epoch": 2.0834868017188457, + "grad_norm": 0.25854116678237915, + "learning_rate": 0.00028055918012376665, + "loss": 0.5038502883911132, + "mean_token_accuracy": 0.8288816407247244, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5468840812146664, + "epoch": 2.2062615101289134, + "grad_norm": 0.35776397585868835, + "learning_rate": 0.00027814486869845917, + "loss": 0.4977967071533203, + "mean_token_accuracy": 0.8320337995886803, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5361524756252766, + "epoch": 2.329036218538981, + "grad_norm": 0.2908971905708313, + "learning_rate": 0.0002754876674222821, + "loss": 0.49344730377197266, + "mean_token_accuracy": 0.8353969538211823, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5515676756203175, + "epoch": 2.4518109269490487, + "grad_norm": 0.2485043853521347, + "learning_rate": 0.0002725924380453484, + "loss": 0.501014518737793, + "mean_token_accuracy": 0.8305466166138649, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5483240431547165, + "epoch": 2.574585635359116, + "grad_norm": 0.27612268924713135, + "learning_rate": 0.000269464477826047, + "loss": 0.4939273834228516, + "mean_token_accuracy": 0.8312122821807861, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5382450266182423, + "epoch": 2.6973603437691835, + "grad_norm": 0.23383453488349915, + "learning_rate": 0.0002661095098389097, + "loss": 0.4967027282714844, + "mean_token_accuracy": 0.8317142990231514, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5448310880362988, + "epoch": 2.820135052179251, + "grad_norm": 0.29418694972991943, + "learning_rate": 0.0002625336725033846, + "loss": 0.4974230194091797, + "mean_token_accuracy": 0.8330811312794686, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.538916184157133, + "epoch": 2.942909760589319, + "grad_norm": 0.2986706495285034, + "learning_rate": 0.0002587435083526725, + "loss": 0.49071075439453127, + "mean_token_accuracy": 0.835035215318203, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5504456080709185, + "eval_mean_token_accuracy": 0.8138916601453509, + "eval_not_syn_loss": 0.5711027383804321, + "eval_not_syn_runtime": 105.759, + "eval_not_syn_samples_per_second": 13.19, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5207725673062461, + "eval_mean_token_accuracy": 0.8279450879778181, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.5415107607841492, + "eval_syn_runtime": 112.382, + "eval_syn_samples_per_second": 12.413, + "eval_syn_steps_per_second": 1.557, + "step": 1224 + }, + { + "entropy": 0.500392770404138, + "epoch": 3.063842848373235, + "grad_norm": 0.24652981758117676, + "learning_rate": 0.0002547459520631764, + "loss": 0.4460899353027344, + "mean_token_accuracy": 0.8447940503885298, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.4625867912173271, + "epoch": 3.1866175567833026, + "grad_norm": 0.2843637466430664, + "learning_rate": 0.00025054831776646675, + "loss": 0.41165496826171877, + "mean_token_accuracy": 0.8544793871045112, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4739935018122196, + "epoch": 3.3093922651933703, + "grad_norm": 0.32995307445526123, + "learning_rate": 0.000246158285666975, + "loss": 0.42344165802001954, + "mean_token_accuracy": 0.8512757349014283, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.47488294705748557, + "epoch": 3.4321669736034375, + "grad_norm": 0.3435416519641876, + "learning_rate": 0.00024158388798990277, + "loss": 0.42080265045166015, + "mean_token_accuracy": 0.8512625667452812, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.4870477955043316, + "epoch": 3.554941682013505, + "grad_norm": 0.33815860748291016, + "learning_rate": 0.0002368334942850555, + "loss": 0.43374271392822267, + "mean_token_accuracy": 0.849637107849121, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.4743228654563427, + "epoch": 3.677716390423573, + "grad_norm": 0.2805832326412201, + "learning_rate": 0.0002319157961134899, + "loss": 0.42299102783203124, + "mean_token_accuracy": 0.85123528778553, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.46706863716244695, + "epoch": 3.80049109883364, + "grad_norm": 0.3060542941093445, + "learning_rate": 0.00022683979114499338, + "loss": 0.4191351318359375, + "mean_token_accuracy": 0.8529838898777962, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4743580640852451, + "epoch": 3.9232658072437077, + "grad_norm": 0.2689199447631836, + "learning_rate": 0.00022161476669549157, + "loss": 0.43219676971435544, + "mean_token_accuracy": 0.8495954439043999, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5012165577071054, + "eval_mean_token_accuracy": 0.8084095290728978, + "eval_not_syn_loss": 0.5803263187408447, + "eval_not_syn_runtime": 105.8265, + "eval_not_syn_samples_per_second": 13.182, + "eval_not_syn_steps_per_second": 1.654, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.47334393518311635, + "eval_mean_token_accuracy": 0.8443782693999154, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5437710881233215, + "eval_syn_runtime": 112.3885, + "eval_syn_samples_per_second": 12.412, + "eval_syn_steps_per_second": 1.557, + "step": 1632 + }, + { + "entropy": 0.4308965259094529, + "epoch": 4.044198895027624, + "grad_norm": 0.3222416043281555, + "learning_rate": 0.0002162502827345043, + "loss": 0.3858341217041016, + "mean_token_accuracy": 0.8619737210612611, + "num_tokens": 4602893.0, + "step": 1650 + }, + { + "entropy": 0.3909018604457378, + "epoch": 4.166973603437691, + "grad_norm": 0.4763369858264923, + "learning_rate": 0.000210756154393741, + "loss": 0.3396052932739258, + "mean_token_accuracy": 0.8764236959815025, + "num_tokens": 4735831.0, + "step": 1700 + }, + { + "entropy": 0.37741987764835355, + "epoch": 4.2897483118477595, + "grad_norm": 0.31223320960998535, + "learning_rate": 0.0002051424340088383, + "loss": 0.33268993377685546, + "mean_token_accuracy": 0.8774440589547158, + "num_tokens": 4880737.0, + "step": 1750 + }, + { + "entropy": 0.38988220423460007, + "epoch": 4.412523020257827, + "grad_norm": 0.3515859842300415, + "learning_rate": 0.00019941939272709723, + "loss": 0.33988418579101565, + "mean_token_accuracy": 0.8746129363775254, + "num_tokens": 5022337.0, + "step": 1800 + }, + { + "entropy": 0.3944451888650656, + "epoch": 4.535297728667894, + "grad_norm": 0.3874817192554474, + "learning_rate": 0.00019359750171487187, + "loss": 0.34360149383544925, + "mean_token_accuracy": 0.8744251477718353, + "num_tokens": 5158628.0, + "step": 1850 + }, + { + "entropy": 0.3910618641972542, + "epoch": 4.658072437077962, + "grad_norm": 0.3768670856952667, + "learning_rate": 0.0001876874129989924, + "loss": 0.34223213195800783, + "mean_token_accuracy": 0.876863748729229, + "num_tokens": 5295686.0, + "step": 1900 + }, + { + "entropy": 0.38558698296546934, + "epoch": 4.780847145488029, + "grad_norm": 0.3657715916633606, + "learning_rate": 0.0001816999399772772, + "loss": 0.3395613861083984, + "mean_token_accuracy": 0.8752773794531822, + "num_tokens": 5440097.0, + "step": 1950 + }, + { + "entropy": 0.3891732354462147, + "epoch": 4.903621853898097, + "grad_norm": 0.2885989844799042, + "learning_rate": 0.00017564603763379236, + "loss": 0.34064876556396484, + "mean_token_accuracy": 0.8750598526000977, + "num_tokens": 5582367.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.43883429646492006, + "eval_mean_token_accuracy": 0.8038648002488272, + "eval_not_syn_loss": 0.6186983585357666, + "eval_not_syn_runtime": 105.7555, + "eval_not_syn_samples_per_second": 13.191, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 5690965.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4145417926992689, + "eval_mean_token_accuracy": 0.8376034893308367, + "eval_num_tokens": 5690965.0, + "eval_syn_loss": 0.5693879127502441, + "eval_syn_runtime": 112.3566, + "eval_syn_samples_per_second": 12.416, + "eval_syn_steps_per_second": 1.558, + "step": 2040 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.25810110441216e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dfa6854aee8f17d1b6fa21e4e34db35f4ef3a40c --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.005046715367502997, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..87c7110e490fb7cf728de598397f8f91cc52a71c --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2448/trainer_state.json @@ -0,0 +1,646 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2448, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8492428332567215, + "epoch": 0.12277470841006753, + "grad_norm": 0.5106982588768005, + "learning_rate": 3.492278846049215e-05, + "loss": 1.7872950744628906, + "mean_token_accuracy": 0.6242395298182964, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8940915194153786, + "epoch": 0.24554941682013506, + "grad_norm": 0.6675613522529602, + "learning_rate": 7.055828688956577e-05, + "loss": 0.8397601318359375, + "mean_token_accuracy": 0.7503150171041488, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7644237650930882, + "epoch": 0.3683241252302026, + "grad_norm": 0.43234843015670776, + "learning_rate": 0.00010619378531863939, + "loss": 0.7161208343505859, + "mean_token_accuracy": 0.778390144109726, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7164452224969864, + "epoch": 0.4910988336402701, + "grad_norm": 0.4049525558948517, + "learning_rate": 0.00014182928374771302, + "loss": 0.6739736938476563, + "mean_token_accuracy": 0.7887251850962639, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7021449965238571, + "epoch": 0.6138735420503376, + "grad_norm": 0.3455510139465332, + "learning_rate": 0.00017746478217678664, + "loss": 0.6584578704833984, + "mean_token_accuracy": 0.7932635286450386, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6882825265824795, + "epoch": 0.7366482504604052, + "grad_norm": 0.30983075499534607, + "learning_rate": 0.00021310028060586025, + "loss": 0.6412281799316406, + "mean_token_accuracy": 0.7957331365346909, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6746629500389099, + "epoch": 0.8594229588704727, + "grad_norm": 0.4235651195049286, + "learning_rate": 0.0002487357790349339, + "loss": 0.6273543167114258, + "mean_token_accuracy": 0.8004380378127098, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6693046718835831, + "epoch": 0.9821976672805403, + "grad_norm": 0.2910408079624176, + "learning_rate": 0.0002843712774640075, + "loss": 0.6247041320800781, + "mean_token_accuracy": 0.8027204623818398, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6973312156541007, + "eval_mean_token_accuracy": 0.7841839187485831, + "eval_not_syn_loss": 0.6392670273780823, + "eval_not_syn_runtime": 106.0995, + "eval_not_syn_samples_per_second": 13.148, + "eval_not_syn_steps_per_second": 1.649, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6523867055347987, + "eval_mean_token_accuracy": 0.8224203096117292, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.5988379120826721, + "eval_syn_runtime": 112.4432, + "eval_syn_samples_per_second": 12.406, + "eval_syn_steps_per_second": 1.556, + "step": 408 + }, + { + "entropy": 0.6396199787026129, + "epoch": 1.1031307550644567, + "grad_norm": 0.3523969054222107, + "learning_rate": 0.0002906962274290906, + "loss": 0.5951201248168946, + "mean_token_accuracy": 0.809070566584011, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6200365445017815, + "epoch": 1.2259054634745243, + "grad_norm": 0.28092390298843384, + "learning_rate": 0.00029034524335317383, + "loss": 0.5773784637451171, + "mean_token_accuracy": 0.8133166086673737, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.62518698990345, + "epoch": 1.3486801718845918, + "grad_norm": 0.34311696887016296, + "learning_rate": 0.0002897290470026443, + "loss": 0.5758082962036133, + "mean_token_accuracy": 0.8123552390933036, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6401302371919155, + "epoch": 1.4714548802946594, + "grad_norm": 0.3884620666503906, + "learning_rate": 0.0002888487658015718, + "loss": 0.58863037109375, + "mean_token_accuracy": 0.8095892634987831, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6160605426132679, + "epoch": 1.5942295887047269, + "grad_norm": 0.34537142515182495, + "learning_rate": 0.00028770601035705147, + "loss": 0.5640074539184571, + "mean_token_accuracy": 0.8150847071409225, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6120209716260433, + "epoch": 1.7170042971147943, + "grad_norm": 0.2542697787284851, + "learning_rate": 0.0002863028715123557, + "loss": 0.5619714736938477, + "mean_token_accuracy": 0.8152382817864418, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6054908482730389, + "epoch": 1.839779005524862, + "grad_norm": 0.3275090456008911, + "learning_rate": 0.0002846419165214213, + "loss": 0.5586128234863281, + "mean_token_accuracy": 0.8170404744148254, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6005898140370846, + "epoch": 1.9625537139349294, + "grad_norm": 0.24491196870803833, + "learning_rate": 0.00028272618435167137, + "loss": 0.5549465179443359, + "mean_token_accuracy": 0.8176508370041847, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6112558654376439, + "eval_mean_token_accuracy": 0.793729545388903, + "eval_not_syn_loss": 0.5950002074241638, + "eval_not_syn_runtime": 105.7692, + "eval_not_syn_samples_per_second": 13.189, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5759283176490239, + "eval_mean_token_accuracy": 0.8401244885580881, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5499612092971802, + "eval_syn_runtime": 112.3691, + "eval_syn_samples_per_second": 12.414, + "eval_syn_steps_per_second": 1.557, + "step": 816 + }, + { + "entropy": 0.5536743695965878, + "epoch": 2.0834868017188457, + "grad_norm": 0.25854116678237915, + "learning_rate": 0.00028055918012376665, + "loss": 0.5038502883911132, + "mean_token_accuracy": 0.8288816407247244, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5468840812146664, + "epoch": 2.2062615101289134, + "grad_norm": 0.35776397585868835, + "learning_rate": 0.00027814486869845917, + "loss": 0.4977967071533203, + "mean_token_accuracy": 0.8320337995886803, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5361524756252766, + "epoch": 2.329036218538981, + "grad_norm": 0.2908971905708313, + "learning_rate": 0.0002754876674222821, + "loss": 0.49344730377197266, + "mean_token_accuracy": 0.8353969538211823, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5515676756203175, + "epoch": 2.4518109269490487, + "grad_norm": 0.2485043853521347, + "learning_rate": 0.0002725924380453484, + "loss": 0.501014518737793, + "mean_token_accuracy": 0.8305466166138649, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5483240431547165, + "epoch": 2.574585635359116, + "grad_norm": 0.27612268924713135, + "learning_rate": 0.000269464477826047, + "loss": 0.4939273834228516, + "mean_token_accuracy": 0.8312122821807861, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5382450266182423, + "epoch": 2.6973603437691835, + "grad_norm": 0.23383453488349915, + "learning_rate": 0.0002661095098389097, + "loss": 0.4967027282714844, + "mean_token_accuracy": 0.8317142990231514, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5448310880362988, + "epoch": 2.820135052179251, + "grad_norm": 0.29418694972991943, + "learning_rate": 0.0002625336725033846, + "loss": 0.4974230194091797, + "mean_token_accuracy": 0.8330811312794686, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.538916184157133, + "epoch": 2.942909760589319, + "grad_norm": 0.2986706495285034, + "learning_rate": 0.0002587435083526725, + "loss": 0.49071075439453127, + "mean_token_accuracy": 0.835035215318203, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5504456080709185, + "eval_mean_token_accuracy": 0.8138916601453509, + "eval_not_syn_loss": 0.5711027383804321, + "eval_not_syn_runtime": 105.759, + "eval_not_syn_samples_per_second": 13.19, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5207725673062461, + "eval_mean_token_accuracy": 0.8279450879778181, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.5415107607841492, + "eval_syn_runtime": 112.382, + "eval_syn_samples_per_second": 12.413, + "eval_syn_steps_per_second": 1.557, + "step": 1224 + }, + { + "entropy": 0.500392770404138, + "epoch": 3.063842848373235, + "grad_norm": 0.24652981758117676, + "learning_rate": 0.0002547459520631764, + "loss": 0.4460899353027344, + "mean_token_accuracy": 0.8447940503885298, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.4625867912173271, + "epoch": 3.1866175567833026, + "grad_norm": 0.2843637466430664, + "learning_rate": 0.00025054831776646675, + "loss": 0.41165496826171877, + "mean_token_accuracy": 0.8544793871045112, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4739935018122196, + "epoch": 3.3093922651933703, + "grad_norm": 0.32995307445526123, + "learning_rate": 0.000246158285666975, + "loss": 0.42344165802001954, + "mean_token_accuracy": 0.8512757349014283, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.47488294705748557, + "epoch": 3.4321669736034375, + "grad_norm": 0.3435416519641876, + "learning_rate": 0.00024158388798990277, + "loss": 0.42080265045166015, + "mean_token_accuracy": 0.8512625667452812, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.4870477955043316, + "epoch": 3.554941682013505, + "grad_norm": 0.33815860748291016, + "learning_rate": 0.0002368334942850555, + "loss": 0.43374271392822267, + "mean_token_accuracy": 0.849637107849121, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.4743228654563427, + "epoch": 3.677716390423573, + "grad_norm": 0.2805832326412201, + "learning_rate": 0.0002319157961134899, + "loss": 0.42299102783203124, + "mean_token_accuracy": 0.85123528778553, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.46706863716244695, + "epoch": 3.80049109883364, + "grad_norm": 0.3060542941093445, + "learning_rate": 0.00022683979114499338, + "loss": 0.4191351318359375, + "mean_token_accuracy": 0.8529838898777962, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4743580640852451, + "epoch": 3.9232658072437077, + "grad_norm": 0.2689199447631836, + "learning_rate": 0.00022161476669549157, + "loss": 0.43219676971435544, + "mean_token_accuracy": 0.8495954439043999, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5012165577071054, + "eval_mean_token_accuracy": 0.8084095290728978, + "eval_not_syn_loss": 0.5803263187408447, + "eval_not_syn_runtime": 105.8265, + "eval_not_syn_samples_per_second": 13.182, + "eval_not_syn_steps_per_second": 1.654, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.47334393518311635, + "eval_mean_token_accuracy": 0.8443782693999154, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5437710881233215, + "eval_syn_runtime": 112.3885, + "eval_syn_samples_per_second": 12.412, + "eval_syn_steps_per_second": 1.557, + "step": 1632 + }, + { + "entropy": 0.4308965259094529, + "epoch": 4.044198895027624, + "grad_norm": 0.3222416043281555, + "learning_rate": 0.0002162502827345043, + "loss": 0.3858341217041016, + "mean_token_accuracy": 0.8619737210612611, + "num_tokens": 4602893.0, + "step": 1650 + }, + { + "entropy": 0.3909018604457378, + "epoch": 4.166973603437691, + "grad_norm": 0.4763369858264923, + "learning_rate": 0.000210756154393741, + "loss": 0.3396052932739258, + "mean_token_accuracy": 0.8764236959815025, + "num_tokens": 4735831.0, + "step": 1700 + }, + { + "entropy": 0.37741987764835355, + "epoch": 4.2897483118477595, + "grad_norm": 0.31223320960998535, + "learning_rate": 0.0002051424340088383, + "loss": 0.33268993377685546, + "mean_token_accuracy": 0.8774440589547158, + "num_tokens": 4880737.0, + "step": 1750 + }, + { + "entropy": 0.38988220423460007, + "epoch": 4.412523020257827, + "grad_norm": 0.3515859842300415, + "learning_rate": 0.00019941939272709723, + "loss": 0.33988418579101565, + "mean_token_accuracy": 0.8746129363775254, + "num_tokens": 5022337.0, + "step": 1800 + }, + { + "entropy": 0.3944451888650656, + "epoch": 4.535297728667894, + "grad_norm": 0.3874817192554474, + "learning_rate": 0.00019359750171487187, + "loss": 0.34360149383544925, + "mean_token_accuracy": 0.8744251477718353, + "num_tokens": 5158628.0, + "step": 1850 + }, + { + "entropy": 0.3910618641972542, + "epoch": 4.658072437077962, + "grad_norm": 0.3768670856952667, + "learning_rate": 0.0001876874129989924, + "loss": 0.34223213195800783, + "mean_token_accuracy": 0.876863748729229, + "num_tokens": 5295686.0, + "step": 1900 + }, + { + "entropy": 0.38558698296546934, + "epoch": 4.780847145488029, + "grad_norm": 0.3657715916633606, + "learning_rate": 0.0001816999399772772, + "loss": 0.3395613861083984, + "mean_token_accuracy": 0.8752773794531822, + "num_tokens": 5440097.0, + "step": 1950 + }, + { + "entropy": 0.3891732354462147, + "epoch": 4.903621853898097, + "grad_norm": 0.2885989844799042, + "learning_rate": 0.00017564603763379236, + "loss": 0.34064876556396484, + "mean_token_accuracy": 0.8750598526000977, + "num_tokens": 5582367.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.43883429646492006, + "eval_mean_token_accuracy": 0.8038648002488272, + "eval_not_syn_loss": 0.6186983585357666, + "eval_not_syn_runtime": 105.7555, + "eval_not_syn_samples_per_second": 13.191, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 5690965.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4145417926992689, + "eval_mean_token_accuracy": 0.8376034893308367, + "eval_num_tokens": 5690965.0, + "eval_syn_loss": 0.5693879127502441, + "eval_syn_runtime": 112.3566, + "eval_syn_samples_per_second": 12.416, + "eval_syn_steps_per_second": 1.558, + "step": 2040 + }, + { + "entropy": 0.37873554607938387, + "epoch": 5.024554941682013, + "grad_norm": 0.516696572303772, + "learning_rate": 0.0001695367824950572, + "loss": 0.32861133575439455, + "mean_token_accuracy": 0.8807547870021181, + "num_tokens": 5718747.0, + "step": 2050 + }, + { + "entropy": 0.30065000750124454, + "epoch": 5.147329650092081, + "grad_norm": 0.39790499210357666, + "learning_rate": 0.00016338335236387066, + "loss": 0.24584257125854492, + "mean_token_accuracy": 0.9059503531455994, + "num_tokens": 5862951.0, + "step": 2100 + }, + { + "entropy": 0.3068667582422495, + "epoch": 5.270104358502149, + "grad_norm": 0.3296535015106201, + "learning_rate": 0.00015719700586783625, + "loss": 0.24911802291870117, + "mean_token_accuracy": 0.9039246591925622, + "num_tokens": 6000342.0, + "step": 2150 + }, + { + "entropy": 0.2965408775210381, + "epoch": 5.392879066912216, + "grad_norm": 0.40667223930358887, + "learning_rate": 0.0001509890618600069, + "loss": 0.2453025436401367, + "mean_token_accuracy": 0.9055307656526566, + "num_tokens": 6145522.0, + "step": 2200 + }, + { + "entropy": 0.30626917764544487, + "epoch": 5.515653775322283, + "grad_norm": 0.3982371687889099, + "learning_rate": 0.00014477087870933735, + "loss": 0.25373943328857423, + "mean_token_accuracy": 0.9033845514059067, + "num_tokens": 6282322.0, + "step": 2250 + }, + { + "entropy": 0.30072322957217695, + "epoch": 5.638428483732351, + "grad_norm": 0.4222176671028137, + "learning_rate": 0.00013855383351883728, + "loss": 0.24919237136840822, + "mean_token_accuracy": 0.9040346756577492, + "num_tokens": 6419205.0, + "step": 2300 + }, + { + "entropy": 0.30633533872663976, + "epoch": 5.7612031921424185, + "grad_norm": 0.3935682475566864, + "learning_rate": 0.00013234930130944655, + "loss": 0.25287656784057616, + "mean_token_accuracy": 0.9037525457143784, + "num_tokens": 6560369.0, + "step": 2350 + }, + { + "entropy": 0.30344139933586123, + "epoch": 5.883977900552486, + "grad_norm": 0.3770838975906372, + "learning_rate": 0.00012616863420772075, + "loss": 0.251728401184082, + "mean_token_accuracy": 0.9035117304325104, + "num_tokens": 6700409.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3781967800004142, + "eval_mean_token_accuracy": 0.8094630067689078, + "eval_not_syn_loss": 0.6938753128051758, + "eval_not_syn_runtime": 105.8435, + "eval_not_syn_samples_per_second": 13.18, + "eval_not_syn_steps_per_second": 1.653, + "eval_num_tokens": 6829158.0, + "step": 2448 + }, + { + "epoch": 6.0, + "eval_entropy": 0.35708206806864057, + "eval_mean_token_accuracy": 0.8336952345711844, + "eval_num_tokens": 6829158.0, + "eval_syn_loss": 0.644233763217926, + "eval_syn_runtime": 112.4137, + "eval_syn_samples_per_second": 12.41, + "eval_syn_steps_per_second": 1.557, + "step": 2448 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.1107534880889754e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dfa6854aee8f17d1b6fa21e4e34db35f4ef3a40c --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.005046715367502997, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ab400a30a499e5f35023adbf47ab9a55e6ca57fc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-2856/trainer_state.json @@ -0,0 +1,758 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2856, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8492428332567215, + "epoch": 0.12277470841006753, + "grad_norm": 0.5106982588768005, + "learning_rate": 3.492278846049215e-05, + "loss": 1.7872950744628906, + "mean_token_accuracy": 0.6242395298182964, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8940915194153786, + "epoch": 0.24554941682013506, + "grad_norm": 0.6675613522529602, + "learning_rate": 7.055828688956577e-05, + "loss": 0.8397601318359375, + "mean_token_accuracy": 0.7503150171041488, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7644237650930882, + "epoch": 0.3683241252302026, + "grad_norm": 0.43234843015670776, + "learning_rate": 0.00010619378531863939, + "loss": 0.7161208343505859, + "mean_token_accuracy": 0.778390144109726, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7164452224969864, + "epoch": 0.4910988336402701, + "grad_norm": 0.4049525558948517, + "learning_rate": 0.00014182928374771302, + "loss": 0.6739736938476563, + "mean_token_accuracy": 0.7887251850962639, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7021449965238571, + "epoch": 0.6138735420503376, + "grad_norm": 0.3455510139465332, + "learning_rate": 0.00017746478217678664, + "loss": 0.6584578704833984, + "mean_token_accuracy": 0.7932635286450386, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6882825265824795, + "epoch": 0.7366482504604052, + "grad_norm": 0.30983075499534607, + "learning_rate": 0.00021310028060586025, + "loss": 0.6412281799316406, + "mean_token_accuracy": 0.7957331365346909, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6746629500389099, + "epoch": 0.8594229588704727, + "grad_norm": 0.4235651195049286, + "learning_rate": 0.0002487357790349339, + "loss": 0.6273543167114258, + "mean_token_accuracy": 0.8004380378127098, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6693046718835831, + "epoch": 0.9821976672805403, + "grad_norm": 0.2910408079624176, + "learning_rate": 0.0002843712774640075, + "loss": 0.6247041320800781, + "mean_token_accuracy": 0.8027204623818398, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6973312156541007, + "eval_mean_token_accuracy": 0.7841839187485831, + "eval_not_syn_loss": 0.6392670273780823, + "eval_not_syn_runtime": 106.0995, + "eval_not_syn_samples_per_second": 13.148, + "eval_not_syn_steps_per_second": 1.649, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6523867055347987, + "eval_mean_token_accuracy": 0.8224203096117292, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.5988379120826721, + "eval_syn_runtime": 112.4432, + "eval_syn_samples_per_second": 12.406, + "eval_syn_steps_per_second": 1.556, + "step": 408 + }, + { + "entropy": 0.6396199787026129, + "epoch": 1.1031307550644567, + "grad_norm": 0.3523969054222107, + "learning_rate": 0.0002906962274290906, + "loss": 0.5951201248168946, + "mean_token_accuracy": 0.809070566584011, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6200365445017815, + "epoch": 1.2259054634745243, + "grad_norm": 0.28092390298843384, + "learning_rate": 0.00029034524335317383, + "loss": 0.5773784637451171, + "mean_token_accuracy": 0.8133166086673737, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.62518698990345, + "epoch": 1.3486801718845918, + "grad_norm": 0.34311696887016296, + "learning_rate": 0.0002897290470026443, + "loss": 0.5758082962036133, + "mean_token_accuracy": 0.8123552390933036, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6401302371919155, + "epoch": 1.4714548802946594, + "grad_norm": 0.3884620666503906, + "learning_rate": 0.0002888487658015718, + "loss": 0.58863037109375, + "mean_token_accuracy": 0.8095892634987831, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6160605426132679, + "epoch": 1.5942295887047269, + "grad_norm": 0.34537142515182495, + "learning_rate": 0.00028770601035705147, + "loss": 0.5640074539184571, + "mean_token_accuracy": 0.8150847071409225, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6120209716260433, + "epoch": 1.7170042971147943, + "grad_norm": 0.2542697787284851, + "learning_rate": 0.0002863028715123557, + "loss": 0.5619714736938477, + "mean_token_accuracy": 0.8152382817864418, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6054908482730389, + "epoch": 1.839779005524862, + "grad_norm": 0.3275090456008911, + "learning_rate": 0.0002846419165214213, + "loss": 0.5586128234863281, + "mean_token_accuracy": 0.8170404744148254, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6005898140370846, + "epoch": 1.9625537139349294, + "grad_norm": 0.24491196870803833, + "learning_rate": 0.00028272618435167137, + "loss": 0.5549465179443359, + "mean_token_accuracy": 0.8176508370041847, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6112558654376439, + "eval_mean_token_accuracy": 0.793729545388903, + "eval_not_syn_loss": 0.5950002074241638, + "eval_not_syn_runtime": 105.7692, + "eval_not_syn_samples_per_second": 13.189, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5759283176490239, + "eval_mean_token_accuracy": 0.8401244885580881, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5499612092971802, + "eval_syn_runtime": 112.3691, + "eval_syn_samples_per_second": 12.414, + "eval_syn_steps_per_second": 1.557, + "step": 816 + }, + { + "entropy": 0.5536743695965878, + "epoch": 2.0834868017188457, + "grad_norm": 0.25854116678237915, + "learning_rate": 0.00028055918012376665, + "loss": 0.5038502883911132, + "mean_token_accuracy": 0.8288816407247244, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5468840812146664, + "epoch": 2.2062615101289134, + "grad_norm": 0.35776397585868835, + "learning_rate": 0.00027814486869845917, + "loss": 0.4977967071533203, + "mean_token_accuracy": 0.8320337995886803, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5361524756252766, + "epoch": 2.329036218538981, + "grad_norm": 0.2908971905708313, + "learning_rate": 0.0002754876674222821, + "loss": 0.49344730377197266, + "mean_token_accuracy": 0.8353969538211823, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5515676756203175, + "epoch": 2.4518109269490487, + "grad_norm": 0.2485043853521347, + "learning_rate": 0.0002725924380453484, + "loss": 0.501014518737793, + "mean_token_accuracy": 0.8305466166138649, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5483240431547165, + "epoch": 2.574585635359116, + "grad_norm": 0.27612268924713135, + "learning_rate": 0.000269464477826047, + "loss": 0.4939273834228516, + "mean_token_accuracy": 0.8312122821807861, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5382450266182423, + "epoch": 2.6973603437691835, + "grad_norm": 0.23383453488349915, + "learning_rate": 0.0002661095098389097, + "loss": 0.4967027282714844, + "mean_token_accuracy": 0.8317142990231514, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5448310880362988, + "epoch": 2.820135052179251, + "grad_norm": 0.29418694972991943, + "learning_rate": 0.0002625336725033846, + "loss": 0.4974230194091797, + "mean_token_accuracy": 0.8330811312794686, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.538916184157133, + "epoch": 2.942909760589319, + "grad_norm": 0.2986706495285034, + "learning_rate": 0.0002587435083526725, + "loss": 0.49071075439453127, + "mean_token_accuracy": 0.835035215318203, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5504456080709185, + "eval_mean_token_accuracy": 0.8138916601453509, + "eval_not_syn_loss": 0.5711027383804321, + "eval_not_syn_runtime": 105.759, + "eval_not_syn_samples_per_second": 13.19, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5207725673062461, + "eval_mean_token_accuracy": 0.8279450879778181, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.5415107607841492, + "eval_syn_runtime": 112.382, + "eval_syn_samples_per_second": 12.413, + "eval_syn_steps_per_second": 1.557, + "step": 1224 + }, + { + "entropy": 0.500392770404138, + "epoch": 3.063842848373235, + "grad_norm": 0.24652981758117676, + "learning_rate": 0.0002547459520631764, + "loss": 0.4460899353027344, + "mean_token_accuracy": 0.8447940503885298, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.4625867912173271, + "epoch": 3.1866175567833026, + "grad_norm": 0.2843637466430664, + "learning_rate": 0.00025054831776646675, + "loss": 0.41165496826171877, + "mean_token_accuracy": 0.8544793871045112, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4739935018122196, + "epoch": 3.3093922651933703, + "grad_norm": 0.32995307445526123, + "learning_rate": 0.000246158285666975, + "loss": 0.42344165802001954, + "mean_token_accuracy": 0.8512757349014283, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.47488294705748557, + "epoch": 3.4321669736034375, + "grad_norm": 0.3435416519641876, + "learning_rate": 0.00024158388798990277, + "loss": 0.42080265045166015, + "mean_token_accuracy": 0.8512625667452812, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.4870477955043316, + "epoch": 3.554941682013505, + "grad_norm": 0.33815860748291016, + "learning_rate": 0.0002368334942850555, + "loss": 0.43374271392822267, + "mean_token_accuracy": 0.849637107849121, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.4743228654563427, + "epoch": 3.677716390423573, + "grad_norm": 0.2805832326412201, + "learning_rate": 0.0002319157961134899, + "loss": 0.42299102783203124, + "mean_token_accuracy": 0.85123528778553, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.46706863716244695, + "epoch": 3.80049109883364, + "grad_norm": 0.3060542941093445, + "learning_rate": 0.00022683979114499338, + "loss": 0.4191351318359375, + "mean_token_accuracy": 0.8529838898777962, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4743580640852451, + "epoch": 3.9232658072437077, + "grad_norm": 0.2689199447631836, + "learning_rate": 0.00022161476669549157, + "loss": 0.43219676971435544, + "mean_token_accuracy": 0.8495954439043999, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5012165577071054, + "eval_mean_token_accuracy": 0.8084095290728978, + "eval_not_syn_loss": 0.5803263187408447, + "eval_not_syn_runtime": 105.8265, + "eval_not_syn_samples_per_second": 13.182, + "eval_not_syn_steps_per_second": 1.654, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.47334393518311635, + "eval_mean_token_accuracy": 0.8443782693999154, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5437710881233215, + "eval_syn_runtime": 112.3885, + "eval_syn_samples_per_second": 12.412, + "eval_syn_steps_per_second": 1.557, + "step": 1632 + }, + { + "entropy": 0.4308965259094529, + "epoch": 4.044198895027624, + "grad_norm": 0.3222416043281555, + "learning_rate": 0.0002162502827345043, + "loss": 0.3858341217041016, + "mean_token_accuracy": 0.8619737210612611, + "num_tokens": 4602893.0, + "step": 1650 + }, + { + "entropy": 0.3909018604457378, + "epoch": 4.166973603437691, + "grad_norm": 0.4763369858264923, + "learning_rate": 0.000210756154393741, + "loss": 0.3396052932739258, + "mean_token_accuracy": 0.8764236959815025, + "num_tokens": 4735831.0, + "step": 1700 + }, + { + "entropy": 0.37741987764835355, + "epoch": 4.2897483118477595, + "grad_norm": 0.31223320960998535, + "learning_rate": 0.0002051424340088383, + "loss": 0.33268993377685546, + "mean_token_accuracy": 0.8774440589547158, + "num_tokens": 4880737.0, + "step": 1750 + }, + { + "entropy": 0.38988220423460007, + "epoch": 4.412523020257827, + "grad_norm": 0.3515859842300415, + "learning_rate": 0.00019941939272709723, + "loss": 0.33988418579101565, + "mean_token_accuracy": 0.8746129363775254, + "num_tokens": 5022337.0, + "step": 1800 + }, + { + "entropy": 0.3944451888650656, + "epoch": 4.535297728667894, + "grad_norm": 0.3874817192554474, + "learning_rate": 0.00019359750171487187, + "loss": 0.34360149383544925, + "mean_token_accuracy": 0.8744251477718353, + "num_tokens": 5158628.0, + "step": 1850 + }, + { + "entropy": 0.3910618641972542, + "epoch": 4.658072437077962, + "grad_norm": 0.3768670856952667, + "learning_rate": 0.0001876874129989924, + "loss": 0.34223213195800783, + "mean_token_accuracy": 0.876863748729229, + "num_tokens": 5295686.0, + "step": 1900 + }, + { + "entropy": 0.38558698296546934, + "epoch": 4.780847145488029, + "grad_norm": 0.3657715916633606, + "learning_rate": 0.0001816999399772772, + "loss": 0.3395613861083984, + "mean_token_accuracy": 0.8752773794531822, + "num_tokens": 5440097.0, + "step": 1950 + }, + { + "entropy": 0.3891732354462147, + "epoch": 4.903621853898097, + "grad_norm": 0.2885989844799042, + "learning_rate": 0.00017564603763379236, + "loss": 0.34064876556396484, + "mean_token_accuracy": 0.8750598526000977, + "num_tokens": 5582367.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.43883429646492006, + "eval_mean_token_accuracy": 0.8038648002488272, + "eval_not_syn_loss": 0.6186983585357666, + "eval_not_syn_runtime": 105.7555, + "eval_not_syn_samples_per_second": 13.191, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 5690965.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4145417926992689, + "eval_mean_token_accuracy": 0.8376034893308367, + "eval_num_tokens": 5690965.0, + "eval_syn_loss": 0.5693879127502441, + "eval_syn_runtime": 112.3566, + "eval_syn_samples_per_second": 12.416, + "eval_syn_steps_per_second": 1.558, + "step": 2040 + }, + { + "entropy": 0.37873554607938387, + "epoch": 5.024554941682013, + "grad_norm": 0.516696572303772, + "learning_rate": 0.0001695367824950572, + "loss": 0.32861133575439455, + "mean_token_accuracy": 0.8807547870021181, + "num_tokens": 5718747.0, + "step": 2050 + }, + { + "entropy": 0.30065000750124454, + "epoch": 5.147329650092081, + "grad_norm": 0.39790499210357666, + "learning_rate": 0.00016338335236387066, + "loss": 0.24584257125854492, + "mean_token_accuracy": 0.9059503531455994, + "num_tokens": 5862951.0, + "step": 2100 + }, + { + "entropy": 0.3068667582422495, + "epoch": 5.270104358502149, + "grad_norm": 0.3296535015106201, + "learning_rate": 0.00015719700586783625, + "loss": 0.24911802291870117, + "mean_token_accuracy": 0.9039246591925622, + "num_tokens": 6000342.0, + "step": 2150 + }, + { + "entropy": 0.2965408775210381, + "epoch": 5.392879066912216, + "grad_norm": 0.40667223930358887, + "learning_rate": 0.0001509890618600069, + "loss": 0.2453025436401367, + "mean_token_accuracy": 0.9055307656526566, + "num_tokens": 6145522.0, + "step": 2200 + }, + { + "entropy": 0.30626917764544487, + "epoch": 5.515653775322283, + "grad_norm": 0.3982371687889099, + "learning_rate": 0.00014477087870933735, + "loss": 0.25373943328857423, + "mean_token_accuracy": 0.9033845514059067, + "num_tokens": 6282322.0, + "step": 2250 + }, + { + "entropy": 0.30072322957217695, + "epoch": 5.638428483732351, + "grad_norm": 0.4222176671028137, + "learning_rate": 0.00013855383351883728, + "loss": 0.24919237136840822, + "mean_token_accuracy": 0.9040346756577492, + "num_tokens": 6419205.0, + "step": 2300 + }, + { + "entropy": 0.30633533872663976, + "epoch": 5.7612031921424185, + "grad_norm": 0.3935682475566864, + "learning_rate": 0.00013234930130944655, + "loss": 0.25287656784057616, + "mean_token_accuracy": 0.9037525457143784, + "num_tokens": 6560369.0, + "step": 2350 + }, + { + "entropy": 0.30344139933586123, + "epoch": 5.883977900552486, + "grad_norm": 0.3770838975906372, + "learning_rate": 0.00012616863420772075, + "loss": 0.251728401184082, + "mean_token_accuracy": 0.9035117304325104, + "num_tokens": 6700409.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3781967800004142, + "eval_mean_token_accuracy": 0.8094630067689078, + "eval_not_syn_loss": 0.6938753128051758, + "eval_not_syn_runtime": 105.8435, + "eval_not_syn_samples_per_second": 13.18, + "eval_not_syn_steps_per_second": 1.653, + "eval_num_tokens": 6829158.0, + "step": 2448 + }, + { + "epoch": 6.0, + "eval_entropy": 0.35708206806864057, + "eval_mean_token_accuracy": 0.8336952345711844, + "eval_num_tokens": 6829158.0, + "eval_syn_loss": 0.644233763217926, + "eval_syn_runtime": 112.4137, + "eval_syn_samples_per_second": 12.41, + "eval_syn_steps_per_second": 1.557, + "step": 2448 + }, + { + "entropy": 0.30334283199709683, + "epoch": 6.004910988336403, + "grad_norm": 0.32158875465393066, + "learning_rate": 0.00012002314067540511, + "loss": 0.24737207412719728, + "mean_token_accuracy": 0.9058935838907504, + "num_tokens": 6834835.0, + "step": 2450 + }, + { + "entropy": 0.2190648239850998, + "epoch": 6.12768569674647, + "grad_norm": 0.3356878459453583, + "learning_rate": 0.00011392406481889967, + "loss": 0.161392879486084, + "mean_token_accuracy": 0.937414693236351, + "num_tokens": 6977951.0, + "step": 2500 + }, + { + "entropy": 0.2200840961188078, + "epoch": 6.250460405156538, + "grad_norm": 0.38168731331825256, + "learning_rate": 0.00010788256581647269, + "loss": 0.16456796646118163, + "mean_token_accuracy": 0.9357201343774796, + "num_tokens": 7121409.0, + "step": 2550 + }, + { + "entropy": 0.2270551733672619, + "epoch": 6.373235113566605, + "grad_norm": 0.43484431505203247, + "learning_rate": 0.00010190969750086217, + "loss": 0.16673795700073243, + "mean_token_accuracy": 0.935126816034317, + "num_tokens": 7258294.0, + "step": 2600 + }, + { + "entropy": 0.22537353567779064, + "epoch": 6.496009821976672, + "grad_norm": 0.3604367673397064, + "learning_rate": 9.601638813462372e-05, + "loss": 0.16410415649414062, + "mean_token_accuracy": 0.9355109342932701, + "num_tokens": 7400869.0, + "step": 2650 + }, + { + "entropy": 0.22000205934047698, + "epoch": 6.6187845303867405, + "grad_norm": 0.3412793278694153, + "learning_rate": 9.02134204152274e-05, + "loss": 0.16420265197753905, + "mean_token_accuracy": 0.9362803927063942, + "num_tokens": 7537988.0, + "step": 2700 + }, + { + "entropy": 0.2240674700215459, + "epoch": 6.741559238796808, + "grad_norm": 0.4121471643447876, + "learning_rate": 8.451141174648785e-05, + "loss": 0.16655830383300782, + "mean_token_accuracy": 0.9351281595230102, + "num_tokens": 7674717.0, + "step": 2750 + }, + { + "entropy": 0.22051083572208882, + "epoch": 6.864333947206875, + "grad_norm": 0.4233345091342926, + "learning_rate": 7.89207948124245e-05, + "loss": 0.16426664352416992, + "mean_token_accuracy": 0.9360027530789375, + "num_tokens": 7809509.0, + "step": 2800 + }, + { + "entropy": 0.21704585194587708, + "epoch": 6.987108655616943, + "grad_norm": 0.3857562243938446, + "learning_rate": 7.345179848909396e-05, + "loss": 0.16032709121704103, + "mean_token_accuracy": 0.9377624034881592, + "num_tokens": 7953622.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.31029051269803726, + "eval_mean_token_accuracy": 0.806178696496146, + "eval_not_syn_loss": 0.8177359104156494, + "eval_not_syn_runtime": 105.7327, + "eval_not_syn_samples_per_second": 13.194, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 7967351.0, + "step": 2856 + }, + { + "epoch": 7.0, + "eval_entropy": 0.2934763271468026, + "eval_mean_token_accuracy": 0.8317537416730608, + "eval_num_tokens": 7967351.0, + "eval_syn_loss": 0.756914496421814, + "eval_syn_runtime": 112.3537, + "eval_syn_samples_per_second": 12.416, + "eval_syn_steps_per_second": 1.558, + "step": 2856 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2949283487417446e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dfa6854aee8f17d1b6fa21e4e34db35f4ef3a40c --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.005046715367502997, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2de50e1e9c2c8c9c98440fdb523e4d31a94894d4 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3264/trainer_state.json @@ -0,0 +1,860 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3264, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8492428332567215, + "epoch": 0.12277470841006753, + "grad_norm": 0.5106982588768005, + "learning_rate": 3.492278846049215e-05, + "loss": 1.7872950744628906, + "mean_token_accuracy": 0.6242395298182964, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8940915194153786, + "epoch": 0.24554941682013506, + "grad_norm": 0.6675613522529602, + "learning_rate": 7.055828688956577e-05, + "loss": 0.8397601318359375, + "mean_token_accuracy": 0.7503150171041488, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7644237650930882, + "epoch": 0.3683241252302026, + "grad_norm": 0.43234843015670776, + "learning_rate": 0.00010619378531863939, + "loss": 0.7161208343505859, + "mean_token_accuracy": 0.778390144109726, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7164452224969864, + "epoch": 0.4910988336402701, + "grad_norm": 0.4049525558948517, + "learning_rate": 0.00014182928374771302, + "loss": 0.6739736938476563, + "mean_token_accuracy": 0.7887251850962639, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7021449965238571, + "epoch": 0.6138735420503376, + "grad_norm": 0.3455510139465332, + "learning_rate": 0.00017746478217678664, + "loss": 0.6584578704833984, + "mean_token_accuracy": 0.7932635286450386, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6882825265824795, + "epoch": 0.7366482504604052, + "grad_norm": 0.30983075499534607, + "learning_rate": 0.00021310028060586025, + "loss": 0.6412281799316406, + "mean_token_accuracy": 0.7957331365346909, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6746629500389099, + "epoch": 0.8594229588704727, + "grad_norm": 0.4235651195049286, + "learning_rate": 0.0002487357790349339, + "loss": 0.6273543167114258, + "mean_token_accuracy": 0.8004380378127098, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6693046718835831, + "epoch": 0.9821976672805403, + "grad_norm": 0.2910408079624176, + "learning_rate": 0.0002843712774640075, + "loss": 0.6247041320800781, + "mean_token_accuracy": 0.8027204623818398, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6973312156541007, + "eval_mean_token_accuracy": 0.7841839187485831, + "eval_not_syn_loss": 0.6392670273780823, + "eval_not_syn_runtime": 106.0995, + "eval_not_syn_samples_per_second": 13.148, + "eval_not_syn_steps_per_second": 1.649, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6523867055347987, + "eval_mean_token_accuracy": 0.8224203096117292, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.5988379120826721, + "eval_syn_runtime": 112.4432, + "eval_syn_samples_per_second": 12.406, + "eval_syn_steps_per_second": 1.556, + "step": 408 + }, + { + "entropy": 0.6396199787026129, + "epoch": 1.1031307550644567, + "grad_norm": 0.3523969054222107, + "learning_rate": 0.0002906962274290906, + "loss": 0.5951201248168946, + "mean_token_accuracy": 0.809070566584011, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6200365445017815, + "epoch": 1.2259054634745243, + "grad_norm": 0.28092390298843384, + "learning_rate": 0.00029034524335317383, + "loss": 0.5773784637451171, + "mean_token_accuracy": 0.8133166086673737, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.62518698990345, + "epoch": 1.3486801718845918, + "grad_norm": 0.34311696887016296, + "learning_rate": 0.0002897290470026443, + "loss": 0.5758082962036133, + "mean_token_accuracy": 0.8123552390933036, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6401302371919155, + "epoch": 1.4714548802946594, + "grad_norm": 0.3884620666503906, + "learning_rate": 0.0002888487658015718, + "loss": 0.58863037109375, + "mean_token_accuracy": 0.8095892634987831, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6160605426132679, + "epoch": 1.5942295887047269, + "grad_norm": 0.34537142515182495, + "learning_rate": 0.00028770601035705147, + "loss": 0.5640074539184571, + "mean_token_accuracy": 0.8150847071409225, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6120209716260433, + "epoch": 1.7170042971147943, + "grad_norm": 0.2542697787284851, + "learning_rate": 0.0002863028715123557, + "loss": 0.5619714736938477, + "mean_token_accuracy": 0.8152382817864418, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6054908482730389, + "epoch": 1.839779005524862, + "grad_norm": 0.3275090456008911, + "learning_rate": 0.0002846419165214213, + "loss": 0.5586128234863281, + "mean_token_accuracy": 0.8170404744148254, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6005898140370846, + "epoch": 1.9625537139349294, + "grad_norm": 0.24491196870803833, + "learning_rate": 0.00028272618435167137, + "loss": 0.5549465179443359, + "mean_token_accuracy": 0.8176508370041847, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6112558654376439, + "eval_mean_token_accuracy": 0.793729545388903, + "eval_not_syn_loss": 0.5950002074241638, + "eval_not_syn_runtime": 105.7692, + "eval_not_syn_samples_per_second": 13.189, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5759283176490239, + "eval_mean_token_accuracy": 0.8401244885580881, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5499612092971802, + "eval_syn_runtime": 112.3691, + "eval_syn_samples_per_second": 12.414, + "eval_syn_steps_per_second": 1.557, + "step": 816 + }, + { + "entropy": 0.5536743695965878, + "epoch": 2.0834868017188457, + "grad_norm": 0.25854116678237915, + "learning_rate": 0.00028055918012376665, + "loss": 0.5038502883911132, + "mean_token_accuracy": 0.8288816407247244, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5468840812146664, + "epoch": 2.2062615101289134, + "grad_norm": 0.35776397585868835, + "learning_rate": 0.00027814486869845917, + "loss": 0.4977967071533203, + "mean_token_accuracy": 0.8320337995886803, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5361524756252766, + "epoch": 2.329036218538981, + "grad_norm": 0.2908971905708313, + "learning_rate": 0.0002754876674222821, + "loss": 0.49344730377197266, + "mean_token_accuracy": 0.8353969538211823, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5515676756203175, + "epoch": 2.4518109269490487, + "grad_norm": 0.2485043853521347, + "learning_rate": 0.0002725924380453484, + "loss": 0.501014518737793, + "mean_token_accuracy": 0.8305466166138649, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5483240431547165, + "epoch": 2.574585635359116, + "grad_norm": 0.27612268924713135, + "learning_rate": 0.000269464477826047, + "loss": 0.4939273834228516, + "mean_token_accuracy": 0.8312122821807861, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5382450266182423, + "epoch": 2.6973603437691835, + "grad_norm": 0.23383453488349915, + "learning_rate": 0.0002661095098389097, + "loss": 0.4967027282714844, + "mean_token_accuracy": 0.8317142990231514, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5448310880362988, + "epoch": 2.820135052179251, + "grad_norm": 0.29418694972991943, + "learning_rate": 0.0002625336725033846, + "loss": 0.4974230194091797, + "mean_token_accuracy": 0.8330811312794686, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.538916184157133, + "epoch": 2.942909760589319, + "grad_norm": 0.2986706495285034, + "learning_rate": 0.0002587435083526725, + "loss": 0.49071075439453127, + "mean_token_accuracy": 0.835035215318203, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5504456080709185, + "eval_mean_token_accuracy": 0.8138916601453509, + "eval_not_syn_loss": 0.5711027383804321, + "eval_not_syn_runtime": 105.759, + "eval_not_syn_samples_per_second": 13.19, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5207725673062461, + "eval_mean_token_accuracy": 0.8279450879778181, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.5415107607841492, + "eval_syn_runtime": 112.382, + "eval_syn_samples_per_second": 12.413, + "eval_syn_steps_per_second": 1.557, + "step": 1224 + }, + { + "entropy": 0.500392770404138, + "epoch": 3.063842848373235, + "grad_norm": 0.24652981758117676, + "learning_rate": 0.0002547459520631764, + "loss": 0.4460899353027344, + "mean_token_accuracy": 0.8447940503885298, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.4625867912173271, + "epoch": 3.1866175567833026, + "grad_norm": 0.2843637466430664, + "learning_rate": 0.00025054831776646675, + "loss": 0.41165496826171877, + "mean_token_accuracy": 0.8544793871045112, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4739935018122196, + "epoch": 3.3093922651933703, + "grad_norm": 0.32995307445526123, + "learning_rate": 0.000246158285666975, + "loss": 0.42344165802001954, + "mean_token_accuracy": 0.8512757349014283, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.47488294705748557, + "epoch": 3.4321669736034375, + "grad_norm": 0.3435416519641876, + "learning_rate": 0.00024158388798990277, + "loss": 0.42080265045166015, + "mean_token_accuracy": 0.8512625667452812, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.4870477955043316, + "epoch": 3.554941682013505, + "grad_norm": 0.33815860748291016, + "learning_rate": 0.0002368334942850555, + "loss": 0.43374271392822267, + "mean_token_accuracy": 0.849637107849121, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.4743228654563427, + "epoch": 3.677716390423573, + "grad_norm": 0.2805832326412201, + "learning_rate": 0.0002319157961134899, + "loss": 0.42299102783203124, + "mean_token_accuracy": 0.85123528778553, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.46706863716244695, + "epoch": 3.80049109883364, + "grad_norm": 0.3060542941093445, + "learning_rate": 0.00022683979114499338, + "loss": 0.4191351318359375, + "mean_token_accuracy": 0.8529838898777962, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4743580640852451, + "epoch": 3.9232658072437077, + "grad_norm": 0.2689199447631836, + "learning_rate": 0.00022161476669549157, + "loss": 0.43219676971435544, + "mean_token_accuracy": 0.8495954439043999, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5012165577071054, + "eval_mean_token_accuracy": 0.8084095290728978, + "eval_not_syn_loss": 0.5803263187408447, + "eval_not_syn_runtime": 105.8265, + "eval_not_syn_samples_per_second": 13.182, + "eval_not_syn_steps_per_second": 1.654, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.47334393518311635, + "eval_mean_token_accuracy": 0.8443782693999154, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5437710881233215, + "eval_syn_runtime": 112.3885, + "eval_syn_samples_per_second": 12.412, + "eval_syn_steps_per_second": 1.557, + "step": 1632 + }, + { + "entropy": 0.4308965259094529, + "epoch": 4.044198895027624, + "grad_norm": 0.3222416043281555, + "learning_rate": 0.0002162502827345043, + "loss": 0.3858341217041016, + "mean_token_accuracy": 0.8619737210612611, + "num_tokens": 4602893.0, + "step": 1650 + }, + { + "entropy": 0.3909018604457378, + "epoch": 4.166973603437691, + "grad_norm": 0.4763369858264923, + "learning_rate": 0.000210756154393741, + "loss": 0.3396052932739258, + "mean_token_accuracy": 0.8764236959815025, + "num_tokens": 4735831.0, + "step": 1700 + }, + { + "entropy": 0.37741987764835355, + "epoch": 4.2897483118477595, + "grad_norm": 0.31223320960998535, + "learning_rate": 0.0002051424340088383, + "loss": 0.33268993377685546, + "mean_token_accuracy": 0.8774440589547158, + "num_tokens": 4880737.0, + "step": 1750 + }, + { + "entropy": 0.38988220423460007, + "epoch": 4.412523020257827, + "grad_norm": 0.3515859842300415, + "learning_rate": 0.00019941939272709723, + "loss": 0.33988418579101565, + "mean_token_accuracy": 0.8746129363775254, + "num_tokens": 5022337.0, + "step": 1800 + }, + { + "entropy": 0.3944451888650656, + "epoch": 4.535297728667894, + "grad_norm": 0.3874817192554474, + "learning_rate": 0.00019359750171487187, + "loss": 0.34360149383544925, + "mean_token_accuracy": 0.8744251477718353, + "num_tokens": 5158628.0, + "step": 1850 + }, + { + "entropy": 0.3910618641972542, + "epoch": 4.658072437077962, + "grad_norm": 0.3768670856952667, + "learning_rate": 0.0001876874129989924, + "loss": 0.34223213195800783, + "mean_token_accuracy": 0.876863748729229, + "num_tokens": 5295686.0, + "step": 1900 + }, + { + "entropy": 0.38558698296546934, + "epoch": 4.780847145488029, + "grad_norm": 0.3657715916633606, + "learning_rate": 0.0001816999399772772, + "loss": 0.3395613861083984, + "mean_token_accuracy": 0.8752773794531822, + "num_tokens": 5440097.0, + "step": 1950 + }, + { + "entropy": 0.3891732354462147, + "epoch": 4.903621853898097, + "grad_norm": 0.2885989844799042, + "learning_rate": 0.00017564603763379236, + "loss": 0.34064876556396484, + "mean_token_accuracy": 0.8750598526000977, + "num_tokens": 5582367.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.43883429646492006, + "eval_mean_token_accuracy": 0.8038648002488272, + "eval_not_syn_loss": 0.6186983585357666, + "eval_not_syn_runtime": 105.7555, + "eval_not_syn_samples_per_second": 13.191, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 5690965.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4145417926992689, + "eval_mean_token_accuracy": 0.8376034893308367, + "eval_num_tokens": 5690965.0, + "eval_syn_loss": 0.5693879127502441, + "eval_syn_runtime": 112.3566, + "eval_syn_samples_per_second": 12.416, + "eval_syn_steps_per_second": 1.558, + "step": 2040 + }, + { + "entropy": 0.37873554607938387, + "epoch": 5.024554941682013, + "grad_norm": 0.516696572303772, + "learning_rate": 0.0001695367824950572, + "loss": 0.32861133575439455, + "mean_token_accuracy": 0.8807547870021181, + "num_tokens": 5718747.0, + "step": 2050 + }, + { + "entropy": 0.30065000750124454, + "epoch": 5.147329650092081, + "grad_norm": 0.39790499210357666, + "learning_rate": 0.00016338335236387066, + "loss": 0.24584257125854492, + "mean_token_accuracy": 0.9059503531455994, + "num_tokens": 5862951.0, + "step": 2100 + }, + { + "entropy": 0.3068667582422495, + "epoch": 5.270104358502149, + "grad_norm": 0.3296535015106201, + "learning_rate": 0.00015719700586783625, + "loss": 0.24911802291870117, + "mean_token_accuracy": 0.9039246591925622, + "num_tokens": 6000342.0, + "step": 2150 + }, + { + "entropy": 0.2965408775210381, + "epoch": 5.392879066912216, + "grad_norm": 0.40667223930358887, + "learning_rate": 0.0001509890618600069, + "loss": 0.2453025436401367, + "mean_token_accuracy": 0.9055307656526566, + "num_tokens": 6145522.0, + "step": 2200 + }, + { + "entropy": 0.30626917764544487, + "epoch": 5.515653775322283, + "grad_norm": 0.3982371687889099, + "learning_rate": 0.00014477087870933735, + "loss": 0.25373943328857423, + "mean_token_accuracy": 0.9033845514059067, + "num_tokens": 6282322.0, + "step": 2250 + }, + { + "entropy": 0.30072322957217695, + "epoch": 5.638428483732351, + "grad_norm": 0.4222176671028137, + "learning_rate": 0.00013855383351883728, + "loss": 0.24919237136840822, + "mean_token_accuracy": 0.9040346756577492, + "num_tokens": 6419205.0, + "step": 2300 + }, + { + "entropy": 0.30633533872663976, + "epoch": 5.7612031921424185, + "grad_norm": 0.3935682475566864, + "learning_rate": 0.00013234930130944655, + "loss": 0.25287656784057616, + "mean_token_accuracy": 0.9037525457143784, + "num_tokens": 6560369.0, + "step": 2350 + }, + { + "entropy": 0.30344139933586123, + "epoch": 5.883977900552486, + "grad_norm": 0.3770838975906372, + "learning_rate": 0.00012616863420772075, + "loss": 0.251728401184082, + "mean_token_accuracy": 0.9035117304325104, + "num_tokens": 6700409.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3781967800004142, + "eval_mean_token_accuracy": 0.8094630067689078, + "eval_not_syn_loss": 0.6938753128051758, + "eval_not_syn_runtime": 105.8435, + "eval_not_syn_samples_per_second": 13.18, + "eval_not_syn_steps_per_second": 1.653, + "eval_num_tokens": 6829158.0, + "step": 2448 + }, + { + "epoch": 6.0, + "eval_entropy": 0.35708206806864057, + "eval_mean_token_accuracy": 0.8336952345711844, + "eval_num_tokens": 6829158.0, + "eval_syn_loss": 0.644233763217926, + "eval_syn_runtime": 112.4137, + "eval_syn_samples_per_second": 12.41, + "eval_syn_steps_per_second": 1.557, + "step": 2448 + }, + { + "entropy": 0.30334283199709683, + "epoch": 6.004910988336403, + "grad_norm": 0.32158875465393066, + "learning_rate": 0.00012002314067540511, + "loss": 0.24737207412719728, + "mean_token_accuracy": 0.9058935838907504, + "num_tokens": 6834835.0, + "step": 2450 + }, + { + "entropy": 0.2190648239850998, + "epoch": 6.12768569674647, + "grad_norm": 0.3356878459453583, + "learning_rate": 0.00011392406481889967, + "loss": 0.161392879486084, + "mean_token_accuracy": 0.937414693236351, + "num_tokens": 6977951.0, + "step": 2500 + }, + { + "entropy": 0.2200840961188078, + "epoch": 6.250460405156538, + "grad_norm": 0.38168731331825256, + "learning_rate": 0.00010788256581647269, + "loss": 0.16456796646118163, + "mean_token_accuracy": 0.9357201343774796, + "num_tokens": 7121409.0, + "step": 2550 + }, + { + "entropy": 0.2270551733672619, + "epoch": 6.373235113566605, + "grad_norm": 0.43484431505203247, + "learning_rate": 0.00010190969750086217, + "loss": 0.16673795700073243, + "mean_token_accuracy": 0.935126816034317, + "num_tokens": 7258294.0, + "step": 2600 + }, + { + "entropy": 0.22537353567779064, + "epoch": 6.496009821976672, + "grad_norm": 0.3604367673397064, + "learning_rate": 9.601638813462372e-05, + "loss": 0.16410415649414062, + "mean_token_accuracy": 0.9355109342932701, + "num_tokens": 7400869.0, + "step": 2650 + }, + { + "entropy": 0.22000205934047698, + "epoch": 6.6187845303867405, + "grad_norm": 0.3412793278694153, + "learning_rate": 9.02134204152274e-05, + "loss": 0.16420265197753905, + "mean_token_accuracy": 0.9362803927063942, + "num_tokens": 7537988.0, + "step": 2700 + }, + { + "entropy": 0.2240674700215459, + "epoch": 6.741559238796808, + "grad_norm": 0.4121471643447876, + "learning_rate": 8.451141174648785e-05, + "loss": 0.16655830383300782, + "mean_token_accuracy": 0.9351281595230102, + "num_tokens": 7674717.0, + "step": 2750 + }, + { + "entropy": 0.22051083572208882, + "epoch": 6.864333947206875, + "grad_norm": 0.4233345091342926, + "learning_rate": 7.89207948124245e-05, + "loss": 0.16426664352416992, + "mean_token_accuracy": 0.9360027530789375, + "num_tokens": 7809509.0, + "step": 2800 + }, + { + "entropy": 0.21704585194587708, + "epoch": 6.987108655616943, + "grad_norm": 0.3857562243938446, + "learning_rate": 7.345179848909396e-05, + "loss": 0.16032709121704103, + "mean_token_accuracy": 0.9377624034881592, + "num_tokens": 7953622.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.31029051269803726, + "eval_mean_token_accuracy": 0.806178696496146, + "eval_not_syn_loss": 0.8177359104156494, + "eval_not_syn_runtime": 105.7327, + "eval_not_syn_samples_per_second": 13.194, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 7967351.0, + "step": 2856 + }, + { + "epoch": 7.0, + "eval_entropy": 0.2934763271468026, + "eval_mean_token_accuracy": 0.8317537416730608, + "eval_num_tokens": 7967351.0, + "eval_syn_loss": 0.756914496421814, + "eval_syn_runtime": 112.3537, + "eval_syn_samples_per_second": 12.416, + "eval_syn_steps_per_second": 1.558, + "step": 2856 + }, + { + "entropy": 0.17915806578954463, + "epoch": 7.108041743400859, + "grad_norm": 0.2797130346298218, + "learning_rate": 6.811442912932006e-05, + "loss": 0.1173903751373291, + "mean_token_accuracy": 0.9555074064259602, + "num_tokens": 8083803.0, + "step": 2900 + }, + { + "entropy": 0.16749983746558428, + "epoch": 7.230816451810927, + "grad_norm": 0.3671954870223999, + "learning_rate": 6.291845225456387e-05, + "loss": 0.10369318008422851, + "mean_token_accuracy": 0.9592780449986458, + "num_tokens": 8226443.0, + "step": 2950 + }, + { + "entropy": 0.16126610741019248, + "epoch": 7.3535911602209945, + "grad_norm": 0.3520691692829132, + "learning_rate": 5.787337468743041e-05, + "loss": 0.10199189186096191, + "mean_token_accuracy": 0.9604821783304215, + "num_tokens": 8368450.0, + "step": 3000 + }, + { + "entropy": 0.15370585326105357, + "epoch": 7.476365868631062, + "grad_norm": 0.4053456485271454, + "learning_rate": 5.298842715750444e-05, + "loss": 0.09939780235290527, + "mean_token_accuracy": 0.9618097496032715, + "num_tokens": 8514445.0, + "step": 3050 + }, + { + "entropy": 0.15956340435892344, + "epoch": 7.59914057704113, + "grad_norm": 0.364583283662796, + "learning_rate": 4.8272547412339716e-05, + "loss": 0.10245522499084472, + "mean_token_accuracy": 0.9601266947388649, + "num_tokens": 8651914.0, + "step": 3100 + }, + { + "entropy": 0.16241536900401116, + "epoch": 7.721915285451197, + "grad_norm": 0.41863277554512024, + "learning_rate": 4.373436386450334e-05, + "loss": 0.10280052185058594, + "mean_token_accuracy": 0.9590485540032386, + "num_tokens": 8793102.0, + "step": 3150 + }, + { + "entropy": 0.16144509088248016, + "epoch": 7.844689993861264, + "grad_norm": 0.35126104950904846, + "learning_rate": 3.938217980459466e-05, + "loss": 0.10448279380798339, + "mean_token_accuracy": 0.95999954611063, + "num_tokens": 8930862.0, + "step": 3200 + }, + { + "entropy": 0.1576566840708256, + "epoch": 7.967464702271332, + "grad_norm": 0.4439404606819153, + "learning_rate": 3.522395820912401e-05, + "loss": 0.10226995468139649, + "mean_token_accuracy": 0.9609322714805603, + "num_tokens": 9068863.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.26425953711782185, + "eval_mean_token_accuracy": 0.8073528463499887, + "eval_not_syn_loss": 0.959676206111908, + "eval_not_syn_runtime": 105.7847, + "eval_not_syn_samples_per_second": 13.187, + "eval_not_syn_steps_per_second": 1.654, + "eval_num_tokens": 9105544.0, + "step": 3264 + }, + { + "epoch": 8.0, + "eval_entropy": 0.24897128105163574, + "eval_mean_token_accuracy": 0.8296991075788226, + "eval_num_tokens": 9105544.0, + "eval_syn_loss": 0.8916293978691101, + "eval_syn_runtime": 112.3751, + "eval_syn_samples_per_second": 12.414, + "eval_syn_steps_per_second": 1.557, + "step": 3264 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.479619450859182e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dfa6854aee8f17d1b6fa21e4e34db35f4ef3a40c --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.005046715367502997, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fc08125685d1c66ad1d9940d61ad4879b48420cd --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-3672/trainer_state.json @@ -0,0 +1,962 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3672, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8492428332567215, + "epoch": 0.12277470841006753, + "grad_norm": 0.5106982588768005, + "learning_rate": 3.492278846049215e-05, + "loss": 1.7872950744628906, + "mean_token_accuracy": 0.6242395298182964, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8940915194153786, + "epoch": 0.24554941682013506, + "grad_norm": 0.6675613522529602, + "learning_rate": 7.055828688956577e-05, + "loss": 0.8397601318359375, + "mean_token_accuracy": 0.7503150171041488, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7644237650930882, + "epoch": 0.3683241252302026, + "grad_norm": 0.43234843015670776, + "learning_rate": 0.00010619378531863939, + "loss": 0.7161208343505859, + "mean_token_accuracy": 0.778390144109726, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7164452224969864, + "epoch": 0.4910988336402701, + "grad_norm": 0.4049525558948517, + "learning_rate": 0.00014182928374771302, + "loss": 0.6739736938476563, + "mean_token_accuracy": 0.7887251850962639, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7021449965238571, + "epoch": 0.6138735420503376, + "grad_norm": 0.3455510139465332, + "learning_rate": 0.00017746478217678664, + "loss": 0.6584578704833984, + "mean_token_accuracy": 0.7932635286450386, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6882825265824795, + "epoch": 0.7366482504604052, + "grad_norm": 0.30983075499534607, + "learning_rate": 0.00021310028060586025, + "loss": 0.6412281799316406, + "mean_token_accuracy": 0.7957331365346909, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6746629500389099, + "epoch": 0.8594229588704727, + "grad_norm": 0.4235651195049286, + "learning_rate": 0.0002487357790349339, + "loss": 0.6273543167114258, + "mean_token_accuracy": 0.8004380378127098, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6693046718835831, + "epoch": 0.9821976672805403, + "grad_norm": 0.2910408079624176, + "learning_rate": 0.0002843712774640075, + "loss": 0.6247041320800781, + "mean_token_accuracy": 0.8027204623818398, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6973312156541007, + "eval_mean_token_accuracy": 0.7841839187485831, + "eval_not_syn_loss": 0.6392670273780823, + "eval_not_syn_runtime": 106.0995, + "eval_not_syn_samples_per_second": 13.148, + "eval_not_syn_steps_per_second": 1.649, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6523867055347987, + "eval_mean_token_accuracy": 0.8224203096117292, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.5988379120826721, + "eval_syn_runtime": 112.4432, + "eval_syn_samples_per_second": 12.406, + "eval_syn_steps_per_second": 1.556, + "step": 408 + }, + { + "entropy": 0.6396199787026129, + "epoch": 1.1031307550644567, + "grad_norm": 0.3523969054222107, + "learning_rate": 0.0002906962274290906, + "loss": 0.5951201248168946, + "mean_token_accuracy": 0.809070566584011, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6200365445017815, + "epoch": 1.2259054634745243, + "grad_norm": 0.28092390298843384, + "learning_rate": 0.00029034524335317383, + "loss": 0.5773784637451171, + "mean_token_accuracy": 0.8133166086673737, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.62518698990345, + "epoch": 1.3486801718845918, + "grad_norm": 0.34311696887016296, + "learning_rate": 0.0002897290470026443, + "loss": 0.5758082962036133, + "mean_token_accuracy": 0.8123552390933036, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6401302371919155, + "epoch": 1.4714548802946594, + "grad_norm": 0.3884620666503906, + "learning_rate": 0.0002888487658015718, + "loss": 0.58863037109375, + "mean_token_accuracy": 0.8095892634987831, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6160605426132679, + "epoch": 1.5942295887047269, + "grad_norm": 0.34537142515182495, + "learning_rate": 0.00028770601035705147, + "loss": 0.5640074539184571, + "mean_token_accuracy": 0.8150847071409225, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6120209716260433, + "epoch": 1.7170042971147943, + "grad_norm": 0.2542697787284851, + "learning_rate": 0.0002863028715123557, + "loss": 0.5619714736938477, + "mean_token_accuracy": 0.8152382817864418, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6054908482730389, + "epoch": 1.839779005524862, + "grad_norm": 0.3275090456008911, + "learning_rate": 0.0002846419165214213, + "loss": 0.5586128234863281, + "mean_token_accuracy": 0.8170404744148254, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6005898140370846, + "epoch": 1.9625537139349294, + "grad_norm": 0.24491196870803833, + "learning_rate": 0.00028272618435167137, + "loss": 0.5549465179443359, + "mean_token_accuracy": 0.8176508370041847, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6112558654376439, + "eval_mean_token_accuracy": 0.793729545388903, + "eval_not_syn_loss": 0.5950002074241638, + "eval_not_syn_runtime": 105.7692, + "eval_not_syn_samples_per_second": 13.189, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5759283176490239, + "eval_mean_token_accuracy": 0.8401244885580881, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5499612092971802, + "eval_syn_runtime": 112.3691, + "eval_syn_samples_per_second": 12.414, + "eval_syn_steps_per_second": 1.557, + "step": 816 + }, + { + "entropy": 0.5536743695965878, + "epoch": 2.0834868017188457, + "grad_norm": 0.25854116678237915, + "learning_rate": 0.00028055918012376665, + "loss": 0.5038502883911132, + "mean_token_accuracy": 0.8288816407247244, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5468840812146664, + "epoch": 2.2062615101289134, + "grad_norm": 0.35776397585868835, + "learning_rate": 0.00027814486869845917, + "loss": 0.4977967071533203, + "mean_token_accuracy": 0.8320337995886803, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5361524756252766, + "epoch": 2.329036218538981, + "grad_norm": 0.2908971905708313, + "learning_rate": 0.0002754876674222821, + "loss": 0.49344730377197266, + "mean_token_accuracy": 0.8353969538211823, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5515676756203175, + "epoch": 2.4518109269490487, + "grad_norm": 0.2485043853521347, + "learning_rate": 0.0002725924380453484, + "loss": 0.501014518737793, + "mean_token_accuracy": 0.8305466166138649, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5483240431547165, + "epoch": 2.574585635359116, + "grad_norm": 0.27612268924713135, + "learning_rate": 0.000269464477826047, + "loss": 0.4939273834228516, + "mean_token_accuracy": 0.8312122821807861, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5382450266182423, + "epoch": 2.6973603437691835, + "grad_norm": 0.23383453488349915, + "learning_rate": 0.0002661095098389097, + "loss": 0.4967027282714844, + "mean_token_accuracy": 0.8317142990231514, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5448310880362988, + "epoch": 2.820135052179251, + "grad_norm": 0.29418694972991943, + "learning_rate": 0.0002625336725033846, + "loss": 0.4974230194091797, + "mean_token_accuracy": 0.8330811312794686, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.538916184157133, + "epoch": 2.942909760589319, + "grad_norm": 0.2986706495285034, + "learning_rate": 0.0002587435083526725, + "loss": 0.49071075439453127, + "mean_token_accuracy": 0.835035215318203, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5504456080709185, + "eval_mean_token_accuracy": 0.8138916601453509, + "eval_not_syn_loss": 0.5711027383804321, + "eval_not_syn_runtime": 105.759, + "eval_not_syn_samples_per_second": 13.19, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5207725673062461, + "eval_mean_token_accuracy": 0.8279450879778181, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.5415107607841492, + "eval_syn_runtime": 112.382, + "eval_syn_samples_per_second": 12.413, + "eval_syn_steps_per_second": 1.557, + "step": 1224 + }, + { + "entropy": 0.500392770404138, + "epoch": 3.063842848373235, + "grad_norm": 0.24652981758117676, + "learning_rate": 0.0002547459520631764, + "loss": 0.4460899353027344, + "mean_token_accuracy": 0.8447940503885298, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.4625867912173271, + "epoch": 3.1866175567833026, + "grad_norm": 0.2843637466430664, + "learning_rate": 0.00025054831776646675, + "loss": 0.41165496826171877, + "mean_token_accuracy": 0.8544793871045112, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4739935018122196, + "epoch": 3.3093922651933703, + "grad_norm": 0.32995307445526123, + "learning_rate": 0.000246158285666975, + "loss": 0.42344165802001954, + "mean_token_accuracy": 0.8512757349014283, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.47488294705748557, + "epoch": 3.4321669736034375, + "grad_norm": 0.3435416519641876, + "learning_rate": 0.00024158388798990277, + "loss": 0.42080265045166015, + "mean_token_accuracy": 0.8512625667452812, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.4870477955043316, + "epoch": 3.554941682013505, + "grad_norm": 0.33815860748291016, + "learning_rate": 0.0002368334942850555, + "loss": 0.43374271392822267, + "mean_token_accuracy": 0.849637107849121, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.4743228654563427, + "epoch": 3.677716390423573, + "grad_norm": 0.2805832326412201, + "learning_rate": 0.0002319157961134899, + "loss": 0.42299102783203124, + "mean_token_accuracy": 0.85123528778553, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.46706863716244695, + "epoch": 3.80049109883364, + "grad_norm": 0.3060542941093445, + "learning_rate": 0.00022683979114499338, + "loss": 0.4191351318359375, + "mean_token_accuracy": 0.8529838898777962, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4743580640852451, + "epoch": 3.9232658072437077, + "grad_norm": 0.2689199447631836, + "learning_rate": 0.00022161476669549157, + "loss": 0.43219676971435544, + "mean_token_accuracy": 0.8495954439043999, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5012165577071054, + "eval_mean_token_accuracy": 0.8084095290728978, + "eval_not_syn_loss": 0.5803263187408447, + "eval_not_syn_runtime": 105.8265, + "eval_not_syn_samples_per_second": 13.182, + "eval_not_syn_steps_per_second": 1.654, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.47334393518311635, + "eval_mean_token_accuracy": 0.8443782693999154, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5437710881233215, + "eval_syn_runtime": 112.3885, + "eval_syn_samples_per_second": 12.412, + "eval_syn_steps_per_second": 1.557, + "step": 1632 + }, + { + "entropy": 0.4308965259094529, + "epoch": 4.044198895027624, + "grad_norm": 0.3222416043281555, + "learning_rate": 0.0002162502827345043, + "loss": 0.3858341217041016, + "mean_token_accuracy": 0.8619737210612611, + "num_tokens": 4602893.0, + "step": 1650 + }, + { + "entropy": 0.3909018604457378, + "epoch": 4.166973603437691, + "grad_norm": 0.4763369858264923, + "learning_rate": 0.000210756154393741, + "loss": 0.3396052932739258, + "mean_token_accuracy": 0.8764236959815025, + "num_tokens": 4735831.0, + "step": 1700 + }, + { + "entropy": 0.37741987764835355, + "epoch": 4.2897483118477595, + "grad_norm": 0.31223320960998535, + "learning_rate": 0.0002051424340088383, + "loss": 0.33268993377685546, + "mean_token_accuracy": 0.8774440589547158, + "num_tokens": 4880737.0, + "step": 1750 + }, + { + "entropy": 0.38988220423460007, + "epoch": 4.412523020257827, + "grad_norm": 0.3515859842300415, + "learning_rate": 0.00019941939272709723, + "loss": 0.33988418579101565, + "mean_token_accuracy": 0.8746129363775254, + "num_tokens": 5022337.0, + "step": 1800 + }, + { + "entropy": 0.3944451888650656, + "epoch": 4.535297728667894, + "grad_norm": 0.3874817192554474, + "learning_rate": 0.00019359750171487187, + "loss": 0.34360149383544925, + "mean_token_accuracy": 0.8744251477718353, + "num_tokens": 5158628.0, + "step": 1850 + }, + { + "entropy": 0.3910618641972542, + "epoch": 4.658072437077962, + "grad_norm": 0.3768670856952667, + "learning_rate": 0.0001876874129989924, + "loss": 0.34223213195800783, + "mean_token_accuracy": 0.876863748729229, + "num_tokens": 5295686.0, + "step": 1900 + }, + { + "entropy": 0.38558698296546934, + "epoch": 4.780847145488029, + "grad_norm": 0.3657715916633606, + "learning_rate": 0.0001816999399772772, + "loss": 0.3395613861083984, + "mean_token_accuracy": 0.8752773794531822, + "num_tokens": 5440097.0, + "step": 1950 + }, + { + "entropy": 0.3891732354462147, + "epoch": 4.903621853898097, + "grad_norm": 0.2885989844799042, + "learning_rate": 0.00017564603763379236, + "loss": 0.34064876556396484, + "mean_token_accuracy": 0.8750598526000977, + "num_tokens": 5582367.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.43883429646492006, + "eval_mean_token_accuracy": 0.8038648002488272, + "eval_not_syn_loss": 0.6186983585357666, + "eval_not_syn_runtime": 105.7555, + "eval_not_syn_samples_per_second": 13.191, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 5690965.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4145417926992689, + "eval_mean_token_accuracy": 0.8376034893308367, + "eval_num_tokens": 5690965.0, + "eval_syn_loss": 0.5693879127502441, + "eval_syn_runtime": 112.3566, + "eval_syn_samples_per_second": 12.416, + "eval_syn_steps_per_second": 1.558, + "step": 2040 + }, + { + "entropy": 0.37873554607938387, + "epoch": 5.024554941682013, + "grad_norm": 0.516696572303772, + "learning_rate": 0.0001695367824950572, + "loss": 0.32861133575439455, + "mean_token_accuracy": 0.8807547870021181, + "num_tokens": 5718747.0, + "step": 2050 + }, + { + "entropy": 0.30065000750124454, + "epoch": 5.147329650092081, + "grad_norm": 0.39790499210357666, + "learning_rate": 0.00016338335236387066, + "loss": 0.24584257125854492, + "mean_token_accuracy": 0.9059503531455994, + "num_tokens": 5862951.0, + "step": 2100 + }, + { + "entropy": 0.3068667582422495, + "epoch": 5.270104358502149, + "grad_norm": 0.3296535015106201, + "learning_rate": 0.00015719700586783625, + "loss": 0.24911802291870117, + "mean_token_accuracy": 0.9039246591925622, + "num_tokens": 6000342.0, + "step": 2150 + }, + { + "entropy": 0.2965408775210381, + "epoch": 5.392879066912216, + "grad_norm": 0.40667223930358887, + "learning_rate": 0.0001509890618600069, + "loss": 0.2453025436401367, + "mean_token_accuracy": 0.9055307656526566, + "num_tokens": 6145522.0, + "step": 2200 + }, + { + "entropy": 0.30626917764544487, + "epoch": 5.515653775322283, + "grad_norm": 0.3982371687889099, + "learning_rate": 0.00014477087870933735, + "loss": 0.25373943328857423, + "mean_token_accuracy": 0.9033845514059067, + "num_tokens": 6282322.0, + "step": 2250 + }, + { + "entropy": 0.30072322957217695, + "epoch": 5.638428483732351, + "grad_norm": 0.4222176671028137, + "learning_rate": 0.00013855383351883728, + "loss": 0.24919237136840822, + "mean_token_accuracy": 0.9040346756577492, + "num_tokens": 6419205.0, + "step": 2300 + }, + { + "entropy": 0.30633533872663976, + "epoch": 5.7612031921424185, + "grad_norm": 0.3935682475566864, + "learning_rate": 0.00013234930130944655, + "loss": 0.25287656784057616, + "mean_token_accuracy": 0.9037525457143784, + "num_tokens": 6560369.0, + "step": 2350 + }, + { + "entropy": 0.30344139933586123, + "epoch": 5.883977900552486, + "grad_norm": 0.3770838975906372, + "learning_rate": 0.00012616863420772075, + "loss": 0.251728401184082, + "mean_token_accuracy": 0.9035117304325104, + "num_tokens": 6700409.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3781967800004142, + "eval_mean_token_accuracy": 0.8094630067689078, + "eval_not_syn_loss": 0.6938753128051758, + "eval_not_syn_runtime": 105.8435, + "eval_not_syn_samples_per_second": 13.18, + "eval_not_syn_steps_per_second": 1.653, + "eval_num_tokens": 6829158.0, + "step": 2448 + }, + { + "epoch": 6.0, + "eval_entropy": 0.35708206806864057, + "eval_mean_token_accuracy": 0.8336952345711844, + "eval_num_tokens": 6829158.0, + "eval_syn_loss": 0.644233763217926, + "eval_syn_runtime": 112.4137, + "eval_syn_samples_per_second": 12.41, + "eval_syn_steps_per_second": 1.557, + "step": 2448 + }, + { + "entropy": 0.30334283199709683, + "epoch": 6.004910988336403, + "grad_norm": 0.32158875465393066, + "learning_rate": 0.00012002314067540511, + "loss": 0.24737207412719728, + "mean_token_accuracy": 0.9058935838907504, + "num_tokens": 6834835.0, + "step": 2450 + }, + { + "entropy": 0.2190648239850998, + "epoch": 6.12768569674647, + "grad_norm": 0.3356878459453583, + "learning_rate": 0.00011392406481889967, + "loss": 0.161392879486084, + "mean_token_accuracy": 0.937414693236351, + "num_tokens": 6977951.0, + "step": 2500 + }, + { + "entropy": 0.2200840961188078, + "epoch": 6.250460405156538, + "grad_norm": 0.38168731331825256, + "learning_rate": 0.00010788256581647269, + "loss": 0.16456796646118163, + "mean_token_accuracy": 0.9357201343774796, + "num_tokens": 7121409.0, + "step": 2550 + }, + { + "entropy": 0.2270551733672619, + "epoch": 6.373235113566605, + "grad_norm": 0.43484431505203247, + "learning_rate": 0.00010190969750086217, + "loss": 0.16673795700073243, + "mean_token_accuracy": 0.935126816034317, + "num_tokens": 7258294.0, + "step": 2600 + }, + { + "entropy": 0.22537353567779064, + "epoch": 6.496009821976672, + "grad_norm": 0.3604367673397064, + "learning_rate": 9.601638813462372e-05, + "loss": 0.16410415649414062, + "mean_token_accuracy": 0.9355109342932701, + "num_tokens": 7400869.0, + "step": 2650 + }, + { + "entropy": 0.22000205934047698, + "epoch": 6.6187845303867405, + "grad_norm": 0.3412793278694153, + "learning_rate": 9.02134204152274e-05, + "loss": 0.16420265197753905, + "mean_token_accuracy": 0.9362803927063942, + "num_tokens": 7537988.0, + "step": 2700 + }, + { + "entropy": 0.2240674700215459, + "epoch": 6.741559238796808, + "grad_norm": 0.4121471643447876, + "learning_rate": 8.451141174648785e-05, + "loss": 0.16655830383300782, + "mean_token_accuracy": 0.9351281595230102, + "num_tokens": 7674717.0, + "step": 2750 + }, + { + "entropy": 0.22051083572208882, + "epoch": 6.864333947206875, + "grad_norm": 0.4233345091342926, + "learning_rate": 7.89207948124245e-05, + "loss": 0.16426664352416992, + "mean_token_accuracy": 0.9360027530789375, + "num_tokens": 7809509.0, + "step": 2800 + }, + { + "entropy": 0.21704585194587708, + "epoch": 6.987108655616943, + "grad_norm": 0.3857562243938446, + "learning_rate": 7.345179848909396e-05, + "loss": 0.16032709121704103, + "mean_token_accuracy": 0.9377624034881592, + "num_tokens": 7953622.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.31029051269803726, + "eval_mean_token_accuracy": 0.806178696496146, + "eval_not_syn_loss": 0.8177359104156494, + "eval_not_syn_runtime": 105.7327, + "eval_not_syn_samples_per_second": 13.194, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 7967351.0, + "step": 2856 + }, + { + "epoch": 7.0, + "eval_entropy": 0.2934763271468026, + "eval_mean_token_accuracy": 0.8317537416730608, + "eval_num_tokens": 7967351.0, + "eval_syn_loss": 0.756914496421814, + "eval_syn_runtime": 112.3537, + "eval_syn_samples_per_second": 12.416, + "eval_syn_steps_per_second": 1.558, + "step": 2856 + }, + { + "entropy": 0.17915806578954463, + "epoch": 7.108041743400859, + "grad_norm": 0.2797130346298218, + "learning_rate": 6.811442912932006e-05, + "loss": 0.1173903751373291, + "mean_token_accuracy": 0.9555074064259602, + "num_tokens": 8083803.0, + "step": 2900 + }, + { + "entropy": 0.16749983746558428, + "epoch": 7.230816451810927, + "grad_norm": 0.3671954870223999, + "learning_rate": 6.291845225456387e-05, + "loss": 0.10369318008422851, + "mean_token_accuracy": 0.9592780449986458, + "num_tokens": 8226443.0, + "step": 2950 + }, + { + "entropy": 0.16126610741019248, + "epoch": 7.3535911602209945, + "grad_norm": 0.3520691692829132, + "learning_rate": 5.787337468743041e-05, + "loss": 0.10199189186096191, + "mean_token_accuracy": 0.9604821783304215, + "num_tokens": 8368450.0, + "step": 3000 + }, + { + "entropy": 0.15370585326105357, + "epoch": 7.476365868631062, + "grad_norm": 0.4053456485271454, + "learning_rate": 5.298842715750444e-05, + "loss": 0.09939780235290527, + "mean_token_accuracy": 0.9618097496032715, + "num_tokens": 8514445.0, + "step": 3050 + }, + { + "entropy": 0.15956340435892344, + "epoch": 7.59914057704113, + "grad_norm": 0.364583283662796, + "learning_rate": 4.8272547412339716e-05, + "loss": 0.10245522499084472, + "mean_token_accuracy": 0.9601266947388649, + "num_tokens": 8651914.0, + "step": 3100 + }, + { + "entropy": 0.16241536900401116, + "epoch": 7.721915285451197, + "grad_norm": 0.41863277554512024, + "learning_rate": 4.373436386450334e-05, + "loss": 0.10280052185058594, + "mean_token_accuracy": 0.9590485540032386, + "num_tokens": 8793102.0, + "step": 3150 + }, + { + "entropy": 0.16144509088248016, + "epoch": 7.844689993861264, + "grad_norm": 0.35126104950904846, + "learning_rate": 3.938217980459466e-05, + "loss": 0.10448279380798339, + "mean_token_accuracy": 0.95999954611063, + "num_tokens": 8930862.0, + "step": 3200 + }, + { + "entropy": 0.1576566840708256, + "epoch": 7.967464702271332, + "grad_norm": 0.4439404606819153, + "learning_rate": 3.522395820912401e-05, + "loss": 0.10226995468139649, + "mean_token_accuracy": 0.9609322714805603, + "num_tokens": 9068863.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.26425953711782185, + "eval_mean_token_accuracy": 0.8073528463499887, + "eval_not_syn_loss": 0.959676206111908, + "eval_not_syn_runtime": 105.7847, + "eval_not_syn_samples_per_second": 13.187, + "eval_not_syn_steps_per_second": 1.654, + "eval_num_tokens": 9105544.0, + "step": 3264 + }, + { + "epoch": 8.0, + "eval_entropy": 0.24897128105163574, + "eval_mean_token_accuracy": 0.8296991075788226, + "eval_num_tokens": 9105544.0, + "eval_syn_loss": 0.8916293978691101, + "eval_syn_runtime": 112.3751, + "eval_syn_samples_per_second": 12.414, + "eval_syn_steps_per_second": 1.557, + "step": 3264 + }, + { + "entropy": 0.14130939807988666, + "epoch": 8.088397790055248, + "grad_norm": 0.21787743270397186, + "learning_rate": 3.126730717104742e-05, + "loss": 0.08074132919311523, + "mean_token_accuracy": 0.9691785347038114, + "num_tokens": 9202212.0, + "step": 3300 + }, + { + "entropy": 0.13008789367973805, + "epoch": 8.211172498465316, + "grad_norm": 0.3095669746398926, + "learning_rate": 2.7519465979614292e-05, + "loss": 0.07130976676940919, + "mean_token_accuracy": 0.9728547465801239, + "num_tokens": 9340486.0, + "step": 3350 + }, + { + "entropy": 0.13310502307489513, + "epoch": 8.333947206875383, + "grad_norm": 0.22650723159313202, + "learning_rate": 2.3987291874996688e-05, + "loss": 0.07299251556396484, + "mean_token_accuracy": 0.9700673633813858, + "num_tokens": 9476614.0, + "step": 3400 + }, + { + "entropy": 0.1311325941979885, + "epoch": 8.456721915285451, + "grad_norm": 0.21754072606563568, + "learning_rate": 2.067724750193533e-05, + "loss": 0.0723969030380249, + "mean_token_accuracy": 0.9714253637194633, + "num_tokens": 9614420.0, + "step": 3450 + }, + { + "entropy": 0.12270729746669531, + "epoch": 8.579496623695519, + "grad_norm": 0.244363933801651, + "learning_rate": 1.7595389085357422e-05, + "loss": 0.06886887550354004, + "mean_token_accuracy": 0.9732170405983925, + "num_tokens": 9764329.0, + "step": 3500 + }, + { + "entropy": 0.13021893870085477, + "epoch": 8.702271332105585, + "grad_norm": 0.2646893262863159, + "learning_rate": 1.4747355349600344e-05, + "loss": 0.07115777969360351, + "mean_token_accuracy": 0.9710743615031242, + "num_tokens": 9907026.0, + "step": 3550 + }, + { + "entropy": 0.1253643373772502, + "epoch": 8.825046040515653, + "grad_norm": 0.21532019972801208, + "learning_rate": 1.2138357201515961e-05, + "loss": 0.07142105102539062, + "mean_token_accuracy": 0.9718041959404945, + "num_tokens": 10048663.0, + "step": 3600 + }, + { + "entropy": 0.1259100575186312, + "epoch": 8.947820748925722, + "grad_norm": 0.27688780426979065, + "learning_rate": 9.773168196331513e-06, + "loss": 0.07123658657073975, + "mean_token_accuracy": 0.9723568171262741, + "num_tokens": 10188058.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.23715950974396297, + "eval_mean_token_accuracy": 0.8104701839174543, + "eval_not_syn_loss": 1.0926613807678223, + "eval_not_syn_runtime": 105.7566, + "eval_not_syn_samples_per_second": 13.191, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 10243737.0, + "step": 3672 + }, + { + "epoch": 9.0, + "eval_entropy": 0.22247669833047048, + "eval_mean_token_accuracy": 0.8235032398360116, + "eval_num_tokens": 10243737.0, + "eval_syn_loss": 1.0430617332458496, + "eval_syn_runtime": 112.3333, + "eval_syn_samples_per_second": 12.418, + "eval_syn_steps_per_second": 1.558, + "step": 3672 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6641560363321242e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dfa6854aee8f17d1b6fa21e4e34db35f4ef3a40c --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.005046715367502997, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6b9550a23829ef96a9157aef63d4a308d37557d2 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/trainer_state.json @@ -0,0 +1,136 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 408, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8492428332567215, + "epoch": 0.12277470841006753, + "grad_norm": 0.5106982588768005, + "learning_rate": 3.492278846049215e-05, + "loss": 1.7872950744628906, + "mean_token_accuracy": 0.6242395298182964, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8940915194153786, + "epoch": 0.24554941682013506, + "grad_norm": 0.6675613522529602, + "learning_rate": 7.055828688956577e-05, + "loss": 0.8397601318359375, + "mean_token_accuracy": 0.7503150171041488, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7644237650930882, + "epoch": 0.3683241252302026, + "grad_norm": 0.43234843015670776, + "learning_rate": 0.00010619378531863939, + "loss": 0.7161208343505859, + "mean_token_accuracy": 0.778390144109726, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7164452224969864, + "epoch": 0.4910988336402701, + "grad_norm": 0.4049525558948517, + "learning_rate": 0.00014182928374771302, + "loss": 0.6739736938476563, + "mean_token_accuracy": 0.7887251850962639, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7021449965238571, + "epoch": 0.6138735420503376, + "grad_norm": 0.3455510139465332, + "learning_rate": 0.00017746478217678664, + "loss": 0.6584578704833984, + "mean_token_accuracy": 0.7932635286450386, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6882825265824795, + "epoch": 0.7366482504604052, + "grad_norm": 0.30983075499534607, + "learning_rate": 0.00021310028060586025, + "loss": 0.6412281799316406, + "mean_token_accuracy": 0.7957331365346909, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6746629500389099, + "epoch": 0.8594229588704727, + "grad_norm": 0.4235651195049286, + "learning_rate": 0.0002487357790349339, + "loss": 0.6273543167114258, + "mean_token_accuracy": 0.8004380378127098, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6693046718835831, + "epoch": 0.9821976672805403, + "grad_norm": 0.2910408079624176, + "learning_rate": 0.0002843712774640075, + "loss": 0.6247041320800781, + "mean_token_accuracy": 0.8027204623818398, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6973312156541007, + "eval_mean_token_accuracy": 0.7841839187485831, + "eval_not_syn_loss": 0.6392670273780823, + "eval_not_syn_runtime": 106.0995, + "eval_not_syn_samples_per_second": 13.148, + "eval_not_syn_steps_per_second": 1.649, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6523867055347987, + "eval_mean_token_accuracy": 0.8224203096117292, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.5988379120826721, + "eval_syn_runtime": 112.4432, + "eval_syn_samples_per_second": 12.406, + "eval_syn_steps_per_second": 1.556, + "step": 408 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.8530281521497088e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dfa6854aee8f17d1b6fa21e4e34db35f4ef3a40c --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.005046715367502997, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..32023b4ac7acaca97954b93178591de57bf2de2f --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/trainer_state.json @@ -0,0 +1,1064 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4080, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8492428332567215, + "epoch": 0.12277470841006753, + "grad_norm": 0.5106982588768005, + "learning_rate": 3.492278846049215e-05, + "loss": 1.7872950744628906, + "mean_token_accuracy": 0.6242395298182964, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8940915194153786, + "epoch": 0.24554941682013506, + "grad_norm": 0.6675613522529602, + "learning_rate": 7.055828688956577e-05, + "loss": 0.8397601318359375, + "mean_token_accuracy": 0.7503150171041488, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7644237650930882, + "epoch": 0.3683241252302026, + "grad_norm": 0.43234843015670776, + "learning_rate": 0.00010619378531863939, + "loss": 0.7161208343505859, + "mean_token_accuracy": 0.778390144109726, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7164452224969864, + "epoch": 0.4910988336402701, + "grad_norm": 0.4049525558948517, + "learning_rate": 0.00014182928374771302, + "loss": 0.6739736938476563, + "mean_token_accuracy": 0.7887251850962639, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7021449965238571, + "epoch": 0.6138735420503376, + "grad_norm": 0.3455510139465332, + "learning_rate": 0.00017746478217678664, + "loss": 0.6584578704833984, + "mean_token_accuracy": 0.7932635286450386, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6882825265824795, + "epoch": 0.7366482504604052, + "grad_norm": 0.30983075499534607, + "learning_rate": 0.00021310028060586025, + "loss": 0.6412281799316406, + "mean_token_accuracy": 0.7957331365346909, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6746629500389099, + "epoch": 0.8594229588704727, + "grad_norm": 0.4235651195049286, + "learning_rate": 0.0002487357790349339, + "loss": 0.6273543167114258, + "mean_token_accuracy": 0.8004380378127098, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6693046718835831, + "epoch": 0.9821976672805403, + "grad_norm": 0.2910408079624176, + "learning_rate": 0.0002843712774640075, + "loss": 0.6247041320800781, + "mean_token_accuracy": 0.8027204623818398, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6973312156541007, + "eval_mean_token_accuracy": 0.7841839187485831, + "eval_not_syn_loss": 0.6392670273780823, + "eval_not_syn_runtime": 106.0995, + "eval_not_syn_samples_per_second": 13.148, + "eval_not_syn_steps_per_second": 1.649, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6523867055347987, + "eval_mean_token_accuracy": 0.8224203096117292, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.5988379120826721, + "eval_syn_runtime": 112.4432, + "eval_syn_samples_per_second": 12.406, + "eval_syn_steps_per_second": 1.556, + "step": 408 + }, + { + "entropy": 0.6396199787026129, + "epoch": 1.1031307550644567, + "grad_norm": 0.3523969054222107, + "learning_rate": 0.0002906962274290906, + "loss": 0.5951201248168946, + "mean_token_accuracy": 0.809070566584011, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6200365445017815, + "epoch": 1.2259054634745243, + "grad_norm": 0.28092390298843384, + "learning_rate": 0.00029034524335317383, + "loss": 0.5773784637451171, + "mean_token_accuracy": 0.8133166086673737, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.62518698990345, + "epoch": 1.3486801718845918, + "grad_norm": 0.34311696887016296, + "learning_rate": 0.0002897290470026443, + "loss": 0.5758082962036133, + "mean_token_accuracy": 0.8123552390933036, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6401302371919155, + "epoch": 1.4714548802946594, + "grad_norm": 0.3884620666503906, + "learning_rate": 0.0002888487658015718, + "loss": 0.58863037109375, + "mean_token_accuracy": 0.8095892634987831, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6160605426132679, + "epoch": 1.5942295887047269, + "grad_norm": 0.34537142515182495, + "learning_rate": 0.00028770601035705147, + "loss": 0.5640074539184571, + "mean_token_accuracy": 0.8150847071409225, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6120209716260433, + "epoch": 1.7170042971147943, + "grad_norm": 0.2542697787284851, + "learning_rate": 0.0002863028715123557, + "loss": 0.5619714736938477, + "mean_token_accuracy": 0.8152382817864418, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6054908482730389, + "epoch": 1.839779005524862, + "grad_norm": 0.3275090456008911, + "learning_rate": 0.0002846419165214213, + "loss": 0.5586128234863281, + "mean_token_accuracy": 0.8170404744148254, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6005898140370846, + "epoch": 1.9625537139349294, + "grad_norm": 0.24491196870803833, + "learning_rate": 0.00028272618435167137, + "loss": 0.5549465179443359, + "mean_token_accuracy": 0.8176508370041847, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6112558654376439, + "eval_mean_token_accuracy": 0.793729545388903, + "eval_not_syn_loss": 0.5950002074241638, + "eval_not_syn_runtime": 105.7692, + "eval_not_syn_samples_per_second": 13.189, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5759283176490239, + "eval_mean_token_accuracy": 0.8401244885580881, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5499612092971802, + "eval_syn_runtime": 112.3691, + "eval_syn_samples_per_second": 12.414, + "eval_syn_steps_per_second": 1.557, + "step": 816 + }, + { + "entropy": 0.5536743695965878, + "epoch": 2.0834868017188457, + "grad_norm": 0.25854116678237915, + "learning_rate": 0.00028055918012376665, + "loss": 0.5038502883911132, + "mean_token_accuracy": 0.8288816407247244, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5468840812146664, + "epoch": 2.2062615101289134, + "grad_norm": 0.35776397585868835, + "learning_rate": 0.00027814486869845917, + "loss": 0.4977967071533203, + "mean_token_accuracy": 0.8320337995886803, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5361524756252766, + "epoch": 2.329036218538981, + "grad_norm": 0.2908971905708313, + "learning_rate": 0.0002754876674222821, + "loss": 0.49344730377197266, + "mean_token_accuracy": 0.8353969538211823, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5515676756203175, + "epoch": 2.4518109269490487, + "grad_norm": 0.2485043853521347, + "learning_rate": 0.0002725924380453484, + "loss": 0.501014518737793, + "mean_token_accuracy": 0.8305466166138649, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5483240431547165, + "epoch": 2.574585635359116, + "grad_norm": 0.27612268924713135, + "learning_rate": 0.000269464477826047, + "loss": 0.4939273834228516, + "mean_token_accuracy": 0.8312122821807861, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5382450266182423, + "epoch": 2.6973603437691835, + "grad_norm": 0.23383453488349915, + "learning_rate": 0.0002661095098389097, + "loss": 0.4967027282714844, + "mean_token_accuracy": 0.8317142990231514, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5448310880362988, + "epoch": 2.820135052179251, + "grad_norm": 0.29418694972991943, + "learning_rate": 0.0002625336725033846, + "loss": 0.4974230194091797, + "mean_token_accuracy": 0.8330811312794686, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.538916184157133, + "epoch": 2.942909760589319, + "grad_norm": 0.2986706495285034, + "learning_rate": 0.0002587435083526725, + "loss": 0.49071075439453127, + "mean_token_accuracy": 0.835035215318203, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5504456080709185, + "eval_mean_token_accuracy": 0.8138916601453509, + "eval_not_syn_loss": 0.5711027383804321, + "eval_not_syn_runtime": 105.759, + "eval_not_syn_samples_per_second": 13.19, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5207725673062461, + "eval_mean_token_accuracy": 0.8279450879778181, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.5415107607841492, + "eval_syn_runtime": 112.382, + "eval_syn_samples_per_second": 12.413, + "eval_syn_steps_per_second": 1.557, + "step": 1224 + }, + { + "entropy": 0.500392770404138, + "epoch": 3.063842848373235, + "grad_norm": 0.24652981758117676, + "learning_rate": 0.0002547459520631764, + "loss": 0.4460899353027344, + "mean_token_accuracy": 0.8447940503885298, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.4625867912173271, + "epoch": 3.1866175567833026, + "grad_norm": 0.2843637466430664, + "learning_rate": 0.00025054831776646675, + "loss": 0.41165496826171877, + "mean_token_accuracy": 0.8544793871045112, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4739935018122196, + "epoch": 3.3093922651933703, + "grad_norm": 0.32995307445526123, + "learning_rate": 0.000246158285666975, + "loss": 0.42344165802001954, + "mean_token_accuracy": 0.8512757349014283, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.47488294705748557, + "epoch": 3.4321669736034375, + "grad_norm": 0.3435416519641876, + "learning_rate": 0.00024158388798990277, + "loss": 0.42080265045166015, + "mean_token_accuracy": 0.8512625667452812, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.4870477955043316, + "epoch": 3.554941682013505, + "grad_norm": 0.33815860748291016, + "learning_rate": 0.0002368334942850555, + "loss": 0.43374271392822267, + "mean_token_accuracy": 0.849637107849121, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.4743228654563427, + "epoch": 3.677716390423573, + "grad_norm": 0.2805832326412201, + "learning_rate": 0.0002319157961134899, + "loss": 0.42299102783203124, + "mean_token_accuracy": 0.85123528778553, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.46706863716244695, + "epoch": 3.80049109883364, + "grad_norm": 0.3060542941093445, + "learning_rate": 0.00022683979114499338, + "loss": 0.4191351318359375, + "mean_token_accuracy": 0.8529838898777962, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4743580640852451, + "epoch": 3.9232658072437077, + "grad_norm": 0.2689199447631836, + "learning_rate": 0.00022161476669549157, + "loss": 0.43219676971435544, + "mean_token_accuracy": 0.8495954439043999, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5012165577071054, + "eval_mean_token_accuracy": 0.8084095290728978, + "eval_not_syn_loss": 0.5803263187408447, + "eval_not_syn_runtime": 105.8265, + "eval_not_syn_samples_per_second": 13.182, + "eval_not_syn_steps_per_second": 1.654, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.47334393518311635, + "eval_mean_token_accuracy": 0.8443782693999154, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5437710881233215, + "eval_syn_runtime": 112.3885, + "eval_syn_samples_per_second": 12.412, + "eval_syn_steps_per_second": 1.557, + "step": 1632 + }, + { + "entropy": 0.4308965259094529, + "epoch": 4.044198895027624, + "grad_norm": 0.3222416043281555, + "learning_rate": 0.0002162502827345043, + "loss": 0.3858341217041016, + "mean_token_accuracy": 0.8619737210612611, + "num_tokens": 4602893.0, + "step": 1650 + }, + { + "entropy": 0.3909018604457378, + "epoch": 4.166973603437691, + "grad_norm": 0.4763369858264923, + "learning_rate": 0.000210756154393741, + "loss": 0.3396052932739258, + "mean_token_accuracy": 0.8764236959815025, + "num_tokens": 4735831.0, + "step": 1700 + }, + { + "entropy": 0.37741987764835355, + "epoch": 4.2897483118477595, + "grad_norm": 0.31223320960998535, + "learning_rate": 0.0002051424340088383, + "loss": 0.33268993377685546, + "mean_token_accuracy": 0.8774440589547158, + "num_tokens": 4880737.0, + "step": 1750 + }, + { + "entropy": 0.38988220423460007, + "epoch": 4.412523020257827, + "grad_norm": 0.3515859842300415, + "learning_rate": 0.00019941939272709723, + "loss": 0.33988418579101565, + "mean_token_accuracy": 0.8746129363775254, + "num_tokens": 5022337.0, + "step": 1800 + }, + { + "entropy": 0.3944451888650656, + "epoch": 4.535297728667894, + "grad_norm": 0.3874817192554474, + "learning_rate": 0.00019359750171487187, + "loss": 0.34360149383544925, + "mean_token_accuracy": 0.8744251477718353, + "num_tokens": 5158628.0, + "step": 1850 + }, + { + "entropy": 0.3910618641972542, + "epoch": 4.658072437077962, + "grad_norm": 0.3768670856952667, + "learning_rate": 0.0001876874129989924, + "loss": 0.34223213195800783, + "mean_token_accuracy": 0.876863748729229, + "num_tokens": 5295686.0, + "step": 1900 + }, + { + "entropy": 0.38558698296546934, + "epoch": 4.780847145488029, + "grad_norm": 0.3657715916633606, + "learning_rate": 0.0001816999399772772, + "loss": 0.3395613861083984, + "mean_token_accuracy": 0.8752773794531822, + "num_tokens": 5440097.0, + "step": 1950 + }, + { + "entropy": 0.3891732354462147, + "epoch": 4.903621853898097, + "grad_norm": 0.2885989844799042, + "learning_rate": 0.00017564603763379236, + "loss": 0.34064876556396484, + "mean_token_accuracy": 0.8750598526000977, + "num_tokens": 5582367.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.43883429646492006, + "eval_mean_token_accuracy": 0.8038648002488272, + "eval_not_syn_loss": 0.6186983585357666, + "eval_not_syn_runtime": 105.7555, + "eval_not_syn_samples_per_second": 13.191, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 5690965.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4145417926992689, + "eval_mean_token_accuracy": 0.8376034893308367, + "eval_num_tokens": 5690965.0, + "eval_syn_loss": 0.5693879127502441, + "eval_syn_runtime": 112.3566, + "eval_syn_samples_per_second": 12.416, + "eval_syn_steps_per_second": 1.558, + "step": 2040 + }, + { + "entropy": 0.37873554607938387, + "epoch": 5.024554941682013, + "grad_norm": 0.516696572303772, + "learning_rate": 0.0001695367824950572, + "loss": 0.32861133575439455, + "mean_token_accuracy": 0.8807547870021181, + "num_tokens": 5718747.0, + "step": 2050 + }, + { + "entropy": 0.30065000750124454, + "epoch": 5.147329650092081, + "grad_norm": 0.39790499210357666, + "learning_rate": 0.00016338335236387066, + "loss": 0.24584257125854492, + "mean_token_accuracy": 0.9059503531455994, + "num_tokens": 5862951.0, + "step": 2100 + }, + { + "entropy": 0.3068667582422495, + "epoch": 5.270104358502149, + "grad_norm": 0.3296535015106201, + "learning_rate": 0.00015719700586783625, + "loss": 0.24911802291870117, + "mean_token_accuracy": 0.9039246591925622, + "num_tokens": 6000342.0, + "step": 2150 + }, + { + "entropy": 0.2965408775210381, + "epoch": 5.392879066912216, + "grad_norm": 0.40667223930358887, + "learning_rate": 0.0001509890618600069, + "loss": 0.2453025436401367, + "mean_token_accuracy": 0.9055307656526566, + "num_tokens": 6145522.0, + "step": 2200 + }, + { + "entropy": 0.30626917764544487, + "epoch": 5.515653775322283, + "grad_norm": 0.3982371687889099, + "learning_rate": 0.00014477087870933735, + "loss": 0.25373943328857423, + "mean_token_accuracy": 0.9033845514059067, + "num_tokens": 6282322.0, + "step": 2250 + }, + { + "entropy": 0.30072322957217695, + "epoch": 5.638428483732351, + "grad_norm": 0.4222176671028137, + "learning_rate": 0.00013855383351883728, + "loss": 0.24919237136840822, + "mean_token_accuracy": 0.9040346756577492, + "num_tokens": 6419205.0, + "step": 2300 + }, + { + "entropy": 0.30633533872663976, + "epoch": 5.7612031921424185, + "grad_norm": 0.3935682475566864, + "learning_rate": 0.00013234930130944655, + "loss": 0.25287656784057616, + "mean_token_accuracy": 0.9037525457143784, + "num_tokens": 6560369.0, + "step": 2350 + }, + { + "entropy": 0.30344139933586123, + "epoch": 5.883977900552486, + "grad_norm": 0.3770838975906372, + "learning_rate": 0.00012616863420772075, + "loss": 0.251728401184082, + "mean_token_accuracy": 0.9035117304325104, + "num_tokens": 6700409.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3781967800004142, + "eval_mean_token_accuracy": 0.8094630067689078, + "eval_not_syn_loss": 0.6938753128051758, + "eval_not_syn_runtime": 105.8435, + "eval_not_syn_samples_per_second": 13.18, + "eval_not_syn_steps_per_second": 1.653, + "eval_num_tokens": 6829158.0, + "step": 2448 + }, + { + "epoch": 6.0, + "eval_entropy": 0.35708206806864057, + "eval_mean_token_accuracy": 0.8336952345711844, + "eval_num_tokens": 6829158.0, + "eval_syn_loss": 0.644233763217926, + "eval_syn_runtime": 112.4137, + "eval_syn_samples_per_second": 12.41, + "eval_syn_steps_per_second": 1.557, + "step": 2448 + }, + { + "entropy": 0.30334283199709683, + "epoch": 6.004910988336403, + "grad_norm": 0.32158875465393066, + "learning_rate": 0.00012002314067540511, + "loss": 0.24737207412719728, + "mean_token_accuracy": 0.9058935838907504, + "num_tokens": 6834835.0, + "step": 2450 + }, + { + "entropy": 0.2190648239850998, + "epoch": 6.12768569674647, + "grad_norm": 0.3356878459453583, + "learning_rate": 0.00011392406481889967, + "loss": 0.161392879486084, + "mean_token_accuracy": 0.937414693236351, + "num_tokens": 6977951.0, + "step": 2500 + }, + { + "entropy": 0.2200840961188078, + "epoch": 6.250460405156538, + "grad_norm": 0.38168731331825256, + "learning_rate": 0.00010788256581647269, + "loss": 0.16456796646118163, + "mean_token_accuracy": 0.9357201343774796, + "num_tokens": 7121409.0, + "step": 2550 + }, + { + "entropy": 0.2270551733672619, + "epoch": 6.373235113566605, + "grad_norm": 0.43484431505203247, + "learning_rate": 0.00010190969750086217, + "loss": 0.16673795700073243, + "mean_token_accuracy": 0.935126816034317, + "num_tokens": 7258294.0, + "step": 2600 + }, + { + "entropy": 0.22537353567779064, + "epoch": 6.496009821976672, + "grad_norm": 0.3604367673397064, + "learning_rate": 9.601638813462372e-05, + "loss": 0.16410415649414062, + "mean_token_accuracy": 0.9355109342932701, + "num_tokens": 7400869.0, + "step": 2650 + }, + { + "entropy": 0.22000205934047698, + "epoch": 6.6187845303867405, + "grad_norm": 0.3412793278694153, + "learning_rate": 9.02134204152274e-05, + "loss": 0.16420265197753905, + "mean_token_accuracy": 0.9362803927063942, + "num_tokens": 7537988.0, + "step": 2700 + }, + { + "entropy": 0.2240674700215459, + "epoch": 6.741559238796808, + "grad_norm": 0.4121471643447876, + "learning_rate": 8.451141174648785e-05, + "loss": 0.16655830383300782, + "mean_token_accuracy": 0.9351281595230102, + "num_tokens": 7674717.0, + "step": 2750 + }, + { + "entropy": 0.22051083572208882, + "epoch": 6.864333947206875, + "grad_norm": 0.4233345091342926, + "learning_rate": 7.89207948124245e-05, + "loss": 0.16426664352416992, + "mean_token_accuracy": 0.9360027530789375, + "num_tokens": 7809509.0, + "step": 2800 + }, + { + "entropy": 0.21704585194587708, + "epoch": 6.987108655616943, + "grad_norm": 0.3857562243938446, + "learning_rate": 7.345179848909396e-05, + "loss": 0.16032709121704103, + "mean_token_accuracy": 0.9377624034881592, + "num_tokens": 7953622.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.31029051269803726, + "eval_mean_token_accuracy": 0.806178696496146, + "eval_not_syn_loss": 0.8177359104156494, + "eval_not_syn_runtime": 105.7327, + "eval_not_syn_samples_per_second": 13.194, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 7967351.0, + "step": 2856 + }, + { + "epoch": 7.0, + "eval_entropy": 0.2934763271468026, + "eval_mean_token_accuracy": 0.8317537416730608, + "eval_num_tokens": 7967351.0, + "eval_syn_loss": 0.756914496421814, + "eval_syn_runtime": 112.3537, + "eval_syn_samples_per_second": 12.416, + "eval_syn_steps_per_second": 1.558, + "step": 2856 + }, + { + "entropy": 0.17915806578954463, + "epoch": 7.108041743400859, + "grad_norm": 0.2797130346298218, + "learning_rate": 6.811442912932006e-05, + "loss": 0.1173903751373291, + "mean_token_accuracy": 0.9555074064259602, + "num_tokens": 8083803.0, + "step": 2900 + }, + { + "entropy": 0.16749983746558428, + "epoch": 7.230816451810927, + "grad_norm": 0.3671954870223999, + "learning_rate": 6.291845225456387e-05, + "loss": 0.10369318008422851, + "mean_token_accuracy": 0.9592780449986458, + "num_tokens": 8226443.0, + "step": 2950 + }, + { + "entropy": 0.16126610741019248, + "epoch": 7.3535911602209945, + "grad_norm": 0.3520691692829132, + "learning_rate": 5.787337468743041e-05, + "loss": 0.10199189186096191, + "mean_token_accuracy": 0.9604821783304215, + "num_tokens": 8368450.0, + "step": 3000 + }, + { + "entropy": 0.15370585326105357, + "epoch": 7.476365868631062, + "grad_norm": 0.4053456485271454, + "learning_rate": 5.298842715750444e-05, + "loss": 0.09939780235290527, + "mean_token_accuracy": 0.9618097496032715, + "num_tokens": 8514445.0, + "step": 3050 + }, + { + "entropy": 0.15956340435892344, + "epoch": 7.59914057704113, + "grad_norm": 0.364583283662796, + "learning_rate": 4.8272547412339716e-05, + "loss": 0.10245522499084472, + "mean_token_accuracy": 0.9601266947388649, + "num_tokens": 8651914.0, + "step": 3100 + }, + { + "entropy": 0.16241536900401116, + "epoch": 7.721915285451197, + "grad_norm": 0.41863277554512024, + "learning_rate": 4.373436386450334e-05, + "loss": 0.10280052185058594, + "mean_token_accuracy": 0.9590485540032386, + "num_tokens": 8793102.0, + "step": 3150 + }, + { + "entropy": 0.16144509088248016, + "epoch": 7.844689993861264, + "grad_norm": 0.35126104950904846, + "learning_rate": 3.938217980459466e-05, + "loss": 0.10448279380798339, + "mean_token_accuracy": 0.95999954611063, + "num_tokens": 8930862.0, + "step": 3200 + }, + { + "entropy": 0.1576566840708256, + "epoch": 7.967464702271332, + "grad_norm": 0.4439404606819153, + "learning_rate": 3.522395820912401e-05, + "loss": 0.10226995468139649, + "mean_token_accuracy": 0.9609322714805603, + "num_tokens": 9068863.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.26425953711782185, + "eval_mean_token_accuracy": 0.8073528463499887, + "eval_not_syn_loss": 0.959676206111908, + "eval_not_syn_runtime": 105.7847, + "eval_not_syn_samples_per_second": 13.187, + "eval_not_syn_steps_per_second": 1.654, + "eval_num_tokens": 9105544.0, + "step": 3264 + }, + { + "epoch": 8.0, + "eval_entropy": 0.24897128105163574, + "eval_mean_token_accuracy": 0.8296991075788226, + "eval_num_tokens": 9105544.0, + "eval_syn_loss": 0.8916293978691101, + "eval_syn_runtime": 112.3751, + "eval_syn_samples_per_second": 12.414, + "eval_syn_steps_per_second": 1.557, + "step": 3264 + }, + { + "entropy": 0.14130939807988666, + "epoch": 8.088397790055248, + "grad_norm": 0.21787743270397186, + "learning_rate": 3.126730717104742e-05, + "loss": 0.08074132919311523, + "mean_token_accuracy": 0.9691785347038114, + "num_tokens": 9202212.0, + "step": 3300 + }, + { + "entropy": 0.13008789367973805, + "epoch": 8.211172498465316, + "grad_norm": 0.3095669746398926, + "learning_rate": 2.7519465979614292e-05, + "loss": 0.07130976676940919, + "mean_token_accuracy": 0.9728547465801239, + "num_tokens": 9340486.0, + "step": 3350 + }, + { + "entropy": 0.13310502307489513, + "epoch": 8.333947206875383, + "grad_norm": 0.22650723159313202, + "learning_rate": 2.3987291874996688e-05, + "loss": 0.07299251556396484, + "mean_token_accuracy": 0.9700673633813858, + "num_tokens": 9476614.0, + "step": 3400 + }, + { + "entropy": 0.1311325941979885, + "epoch": 8.456721915285451, + "grad_norm": 0.21754072606563568, + "learning_rate": 2.067724750193533e-05, + "loss": 0.0723969030380249, + "mean_token_accuracy": 0.9714253637194633, + "num_tokens": 9614420.0, + "step": 3450 + }, + { + "entropy": 0.12270729746669531, + "epoch": 8.579496623695519, + "grad_norm": 0.244363933801651, + "learning_rate": 1.7595389085357422e-05, + "loss": 0.06886887550354004, + "mean_token_accuracy": 0.9732170405983925, + "num_tokens": 9764329.0, + "step": 3500 + }, + { + "entropy": 0.13021893870085477, + "epoch": 8.702271332105585, + "grad_norm": 0.2646893262863159, + "learning_rate": 1.4747355349600344e-05, + "loss": 0.07115777969360351, + "mean_token_accuracy": 0.9710743615031242, + "num_tokens": 9907026.0, + "step": 3550 + }, + { + "entropy": 0.1253643373772502, + "epoch": 8.825046040515653, + "grad_norm": 0.21532019972801208, + "learning_rate": 1.2138357201515961e-05, + "loss": 0.07142105102539062, + "mean_token_accuracy": 0.9718041959404945, + "num_tokens": 10048663.0, + "step": 3600 + }, + { + "entropy": 0.1259100575186312, + "epoch": 8.947820748925722, + "grad_norm": 0.27688780426979065, + "learning_rate": 9.773168196331513e-06, + "loss": 0.07123658657073975, + "mean_token_accuracy": 0.9723568171262741, + "num_tokens": 10188058.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.23715950974396297, + "eval_mean_token_accuracy": 0.8104701839174543, + "eval_not_syn_loss": 1.0926613807678223, + "eval_not_syn_runtime": 105.7566, + "eval_not_syn_samples_per_second": 13.191, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 10243737.0, + "step": 3672 + }, + { + "epoch": 9.0, + "eval_entropy": 0.22247669833047048, + "eval_mean_token_accuracy": 0.8235032398360116, + "eval_num_tokens": 10243737.0, + "eval_syn_loss": 1.0430617332458496, + "eval_syn_runtime": 112.3333, + "eval_syn_samples_per_second": 12.418, + "eval_syn_steps_per_second": 1.558, + "step": 3672 + }, + { + "entropy": 0.12065082047191368, + "epoch": 9.068753836709638, + "grad_norm": 0.22517003118991852, + "learning_rate": 7.65611580371088e-06, + "loss": 0.0654967737197876, + "mean_token_accuracy": 0.9752966471129868, + "num_tokens": 10325246.0, + "step": 3700 + }, + { + "entropy": 0.12137452559545636, + "epoch": 9.191528545119706, + "grad_norm": 0.24532213807106018, + "learning_rate": 5.791073489996836e-06, + "loss": 0.06237410068511963, + "mean_token_accuracy": 0.975640364587307, + "num_tokens": 10460259.0, + "step": 3750 + }, + { + "entropy": 0.11854210389778018, + "epoch": 9.314303253529772, + "grad_norm": 0.20312942564487457, + "learning_rate": 4.18145363112073e-06, + "loss": 0.062231740951538085, + "mean_token_accuracy": 0.9759511294960975, + "num_tokens": 10596880.0, + "step": 3800 + }, + { + "entropy": 0.11403276169672609, + "epoch": 9.43707796193984, + "grad_norm": 0.19876134395599365, + "learning_rate": 2.8302012691466577e-06, + "loss": 0.06135953426361084, + "mean_token_accuracy": 0.9764540281891823, + "num_tokens": 10737950.0, + "step": 3850 + }, + { + "entropy": 0.12379559995606541, + "epoch": 9.559852670349908, + "grad_norm": 0.20855098962783813, + "learning_rate": 1.7397887238730806e-06, + "loss": 0.06396660804748536, + "mean_token_accuracy": 0.9747681060433387, + "num_tokens": 10873020.0, + "step": 3900 + }, + { + "entropy": 0.11402799531817437, + "epoch": 9.682627378759975, + "grad_norm": 0.14763331413269043, + "learning_rate": 9.122110693513342e-07, + "loss": 0.059934263229370115, + "mean_token_accuracy": 0.9764754155278206, + "num_tokens": 11014972.0, + "step": 3950 + }, + { + "entropy": 0.114491401091218, + "epoch": 9.805402087170043, + "grad_norm": 0.1506061553955078, + "learning_rate": 3.489824835970767e-07, + "loss": 0.060681548118591305, + "mean_token_accuracy": 0.9762310767173767, + "num_tokens": 11156776.0, + "step": 4000 + }, + { + "entropy": 0.11396890021860599, + "epoch": 9.92817679558011, + "grad_norm": 0.19503675401210785, + "learning_rate": 5.1133478173345325e-08, + "loss": 0.059567413330078124, + "mean_token_accuracy": 0.97712103754282, + "num_tokens": 11301352.0, + "step": 4050 + }, + { + "epoch": 10.0, + "eval_entropy": 0.22446906200477054, + "eval_mean_token_accuracy": 0.8108472946711949, + "eval_not_syn_loss": 1.170738697052002, + "eval_not_syn_runtime": 105.7553, + "eval_not_syn_samples_per_second": 13.191, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 11381930.0, + "step": 4080 + }, + { + "epoch": 10.0, + "eval_entropy": 0.21062756402151925, + "eval_mean_token_accuracy": 0.8237310927254813, + "eval_num_tokens": 11381930.0, + "eval_syn_loss": 1.1185699701309204, + "eval_syn_runtime": 112.3077, + "eval_syn_samples_per_second": 12.421, + "eval_syn_steps_per_second": 1.558, + "step": 4080 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.8489438807785165e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dfa6854aee8f17d1b6fa21e4e34db35f4ef3a40c --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.005046715367502997, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3002d6d65d31095592ac89570c870e64e8aa2f80 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/trainer_state.json @@ -0,0 +1,238 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 816, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8492428332567215, + "epoch": 0.12277470841006753, + "grad_norm": 0.5106982588768005, + "learning_rate": 3.492278846049215e-05, + "loss": 1.7872950744628906, + "mean_token_accuracy": 0.6242395298182964, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8940915194153786, + "epoch": 0.24554941682013506, + "grad_norm": 0.6675613522529602, + "learning_rate": 7.055828688956577e-05, + "loss": 0.8397601318359375, + "mean_token_accuracy": 0.7503150171041488, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7644237650930882, + "epoch": 0.3683241252302026, + "grad_norm": 0.43234843015670776, + "learning_rate": 0.00010619378531863939, + "loss": 0.7161208343505859, + "mean_token_accuracy": 0.778390144109726, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7164452224969864, + "epoch": 0.4910988336402701, + "grad_norm": 0.4049525558948517, + "learning_rate": 0.00014182928374771302, + "loss": 0.6739736938476563, + "mean_token_accuracy": 0.7887251850962639, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7021449965238571, + "epoch": 0.6138735420503376, + "grad_norm": 0.3455510139465332, + "learning_rate": 0.00017746478217678664, + "loss": 0.6584578704833984, + "mean_token_accuracy": 0.7932635286450386, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6882825265824795, + "epoch": 0.7366482504604052, + "grad_norm": 0.30983075499534607, + "learning_rate": 0.00021310028060586025, + "loss": 0.6412281799316406, + "mean_token_accuracy": 0.7957331365346909, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6746629500389099, + "epoch": 0.8594229588704727, + "grad_norm": 0.4235651195049286, + "learning_rate": 0.0002487357790349339, + "loss": 0.6273543167114258, + "mean_token_accuracy": 0.8004380378127098, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6693046718835831, + "epoch": 0.9821976672805403, + "grad_norm": 0.2910408079624176, + "learning_rate": 0.0002843712774640075, + "loss": 0.6247041320800781, + "mean_token_accuracy": 0.8027204623818398, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6973312156541007, + "eval_mean_token_accuracy": 0.7841839187485831, + "eval_not_syn_loss": 0.6392670273780823, + "eval_not_syn_runtime": 106.0995, + "eval_not_syn_samples_per_second": 13.148, + "eval_not_syn_steps_per_second": 1.649, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6523867055347987, + "eval_mean_token_accuracy": 0.8224203096117292, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.5988379120826721, + "eval_syn_runtime": 112.4432, + "eval_syn_samples_per_second": 12.406, + "eval_syn_steps_per_second": 1.556, + "step": 408 + }, + { + "entropy": 0.6396199787026129, + "epoch": 1.1031307550644567, + "grad_norm": 0.3523969054222107, + "learning_rate": 0.0002906962274290906, + "loss": 0.5951201248168946, + "mean_token_accuracy": 0.809070566584011, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6200365445017815, + "epoch": 1.2259054634745243, + "grad_norm": 0.28092390298843384, + "learning_rate": 0.00029034524335317383, + "loss": 0.5773784637451171, + "mean_token_accuracy": 0.8133166086673737, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.62518698990345, + "epoch": 1.3486801718845918, + "grad_norm": 0.34311696887016296, + "learning_rate": 0.0002897290470026443, + "loss": 0.5758082962036133, + "mean_token_accuracy": 0.8123552390933036, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6401302371919155, + "epoch": 1.4714548802946594, + "grad_norm": 0.3884620666503906, + "learning_rate": 0.0002888487658015718, + "loss": 0.58863037109375, + "mean_token_accuracy": 0.8095892634987831, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6160605426132679, + "epoch": 1.5942295887047269, + "grad_norm": 0.34537142515182495, + "learning_rate": 0.00028770601035705147, + "loss": 0.5640074539184571, + "mean_token_accuracy": 0.8150847071409225, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6120209716260433, + "epoch": 1.7170042971147943, + "grad_norm": 0.2542697787284851, + "learning_rate": 0.0002863028715123557, + "loss": 0.5619714736938477, + "mean_token_accuracy": 0.8152382817864418, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6054908482730389, + "epoch": 1.839779005524862, + "grad_norm": 0.3275090456008911, + "learning_rate": 0.0002846419165214213, + "loss": 0.5586128234863281, + "mean_token_accuracy": 0.8170404744148254, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6005898140370846, + "epoch": 1.9625537139349294, + "grad_norm": 0.24491196870803833, + "learning_rate": 0.00028272618435167137, + "loss": 0.5549465179443359, + "mean_token_accuracy": 0.8176508370041847, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6112558654376439, + "eval_mean_token_accuracy": 0.793729545388903, + "eval_not_syn_loss": 0.5950002074241638, + "eval_not_syn_runtime": 105.7692, + "eval_not_syn_samples_per_second": 13.189, + "eval_not_syn_steps_per_second": 1.655, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5759283176490239, + "eval_mean_token_accuracy": 0.8401244885580881, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5499612092971802, + "eval_syn_runtime": 112.3691, + "eval_syn_samples_per_second": 12.414, + "eval_syn_steps_per_second": 1.557, + "step": 816 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.701594466763469e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9118eb8ff2c647ac8f1acb1826eb1cd73893fe31 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.06743035903930279, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..02cbf45c304a7c0fd1a218990139eb18ad058221 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/trainer_state.json @@ -0,0 +1,758 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2856, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.84347476541996, + "epoch": 0.12277470841006753, + "grad_norm": 0.5139632821083069, + "learning_rate": 3.586242144074487e-05, + "loss": 1.7810469055175782, + "mean_token_accuracy": 0.6252794374525547, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8905406814813613, + "epoch": 0.24554941682013506, + "grad_norm": 0.6561369299888611, + "learning_rate": 7.245672903334166e-05, + "loss": 0.83647705078125, + "mean_token_accuracy": 0.7502138108015061, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7635939045250416, + "epoch": 0.3683241252302026, + "grad_norm": 0.4626609683036804, + "learning_rate": 0.00010905103662593846, + "loss": 0.7156005096435547, + "mean_token_accuracy": 0.7786111453175545, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7165373960137367, + "epoch": 0.4910988336402701, + "grad_norm": 0.4098263680934906, + "learning_rate": 0.00014564534421853526, + "loss": 0.67410400390625, + "mean_token_accuracy": 0.7880693352222443, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7035001173615456, + "epoch": 0.6138735420503376, + "grad_norm": 0.33693116903305054, + "learning_rate": 0.0001822396518111321, + "loss": 0.6591246032714844, + "mean_token_accuracy": 0.7931166198849678, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6862310113012791, + "epoch": 0.7366482504604052, + "grad_norm": 0.31358352303504944, + "learning_rate": 0.00021883395940372884, + "loss": 0.6399201202392578, + "mean_token_accuracy": 0.7952846321463585, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6739526629447937, + "epoch": 0.8594229588704727, + "grad_norm": 0.4029310941696167, + "learning_rate": 0.00025542826699632564, + "loss": 0.6259587860107422, + "mean_token_accuracy": 0.8014272648096085, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6681017802655697, + "epoch": 0.9821976672805403, + "grad_norm": 0.3852674961090088, + "learning_rate": 0.00029202257458892247, + "loss": 0.6253739166259765, + "mean_token_accuracy": 0.8020894029736518, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6994867869785854, + "eval_mean_token_accuracy": 0.7849237663405282, + "eval_not_syn_loss": 0.6411319971084595, + "eval_not_syn_runtime": 110.7069, + "eval_not_syn_samples_per_second": 12.61, + "eval_not_syn_steps_per_second": 1.581, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6563669620241438, + "eval_mean_token_accuracy": 0.8207755245481219, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.6019997000694275, + "eval_syn_runtime": 117.5102, + "eval_syn_samples_per_second": 11.88, + "eval_syn_steps_per_second": 1.489, + "step": 408 + }, + { + "entropy": 0.6358311531809986, + "epoch": 1.1031307550644567, + "grad_norm": 0.34651753306388855, + "learning_rate": 0.00029851770373635705, + "loss": 0.5962887191772461, + "mean_token_accuracy": 0.8082216354796123, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6196171633899212, + "epoch": 1.2259054634745243, + "grad_norm": 0.28438520431518555, + "learning_rate": 0.0002981572760785325, + "loss": 0.5779812622070313, + "mean_token_accuracy": 0.8130937224626541, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.624696860164404, + "epoch": 1.3486801718845918, + "grad_norm": 0.3563199043273926, + "learning_rate": 0.000297524500341339, + "loss": 0.5760621643066406, + "mean_token_accuracy": 0.8123102071881294, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6383913996815681, + "epoch": 1.4714548802946594, + "grad_norm": 0.32306408882141113, + "learning_rate": 0.00029662053428333246, + "loss": 0.5874852371215821, + "mean_token_accuracy": 0.8098450502753258, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6132303845882415, + "epoch": 1.5942295887047269, + "grad_norm": 0.3351016044616699, + "learning_rate": 0.0002954470318466225, + "loss": 0.5624140930175782, + "mean_token_accuracy": 0.8152360209822654, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6104247760772705, + "epoch": 1.7170042971147943, + "grad_norm": 0.25214216113090515, + "learning_rate": 0.00029400614013073664, + "loss": 0.5608636093139648, + "mean_token_accuracy": 0.8154580116271972, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6093999670445919, + "epoch": 1.839779005524862, + "grad_norm": 0.25383421778678894, + "learning_rate": 0.0002923004954641779, + "loss": 0.5591114044189454, + "mean_token_accuracy": 0.817192807495594, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6029203486442566, + "epoch": 1.9625537139349294, + "grad_norm": 0.25239020586013794, + "learning_rate": 0.0002903332185808646, + "loss": 0.5529713821411133, + "mean_token_accuracy": 0.8182568901777267, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6090937239783151, + "eval_mean_token_accuracy": 0.7930077845709664, + "eval_not_syn_loss": 0.5995497107505798, + "eval_not_syn_runtime": 110.3965, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5718999467577253, + "eval_mean_token_accuracy": 0.84101799760546, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5510491728782654, + "eval_syn_runtime": 117.3523, + "eval_syn_samples_per_second": 11.896, + "eval_syn_steps_per_second": 1.491, + "step": 816 + }, + { + "entropy": 0.5504116184517817, + "epoch": 2.0834868017188457, + "grad_norm": 0.23869574069976807, + "learning_rate": 0.0002881079089102777, + "loss": 0.5012085723876953, + "mean_token_accuracy": 0.8299948473267144, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5487727333605289, + "epoch": 2.2062615101289134, + "grad_norm": 0.3087007403373718, + "learning_rate": 0.00028562863799176175, + "loss": 0.4988512802124023, + "mean_token_accuracy": 0.8319525212049484, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5354658082127571, + "epoch": 2.329036218538981, + "grad_norm": 0.2878682613372803, + "learning_rate": 0.00028289994202503066, + "loss": 0.4919636917114258, + "mean_token_accuracy": 0.8349191680550575, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5529813665151596, + "epoch": 2.4518109269490487, + "grad_norm": 0.2625614106655121, + "learning_rate": 0.00027992681357050643, + "loss": 0.5027856063842774, + "mean_token_accuracy": 0.8296491304039955, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5497122646868229, + "epoch": 2.574585635359116, + "grad_norm": 0.271993488073349, + "learning_rate": 0.00027671469241467785, + "loss": 0.49415691375732423, + "mean_token_accuracy": 0.8312779009342194, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5419583600759507, + "epoch": 2.6973603437691835, + "grad_norm": 0.23863548040390015, + "learning_rate": 0.00027326945561719136, + "loss": 0.49570159912109374, + "mean_token_accuracy": 0.8327477470040321, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5491574917733669, + "epoch": 2.820135052179251, + "grad_norm": 0.29003486037254333, + "learning_rate": 0.00026959740675788486, + "loss": 0.4965015411376953, + "mean_token_accuracy": 0.8336276519298553, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.5377901926636696, + "epoch": 2.942909760589319, + "grad_norm": 0.27723413705825806, + "learning_rate": 0.0002657052644034388, + "loss": 0.48964527130126956, + "mean_token_accuracy": 0.8356471425294876, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5576562706061772, + "eval_mean_token_accuracy": 0.817173547404153, + "eval_not_syn_loss": 0.5734513401985168, + "eval_not_syn_runtime": 110.4329, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5267024655001504, + "eval_mean_token_accuracy": 0.8285774500029428, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.544999361038208, + "eval_syn_runtime": 117.3289, + "eval_syn_samples_per_second": 11.898, + "eval_syn_steps_per_second": 1.492, + "step": 1224 + }, + { + "entropy": 0.502170312676938, + "epoch": 3.063842848373235, + "grad_norm": 0.44433876872062683, + "learning_rate": 0.0002616001498147458, + "loss": 0.448189811706543, + "mean_token_accuracy": 0.8443391725496592, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.46903550207614897, + "epoch": 3.1866175567833026, + "grad_norm": 0.2440977245569229, + "learning_rate": 0.0002572895739174909, + "loss": 0.4171265029907227, + "mean_token_accuracy": 0.8530975985527038, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4750825077295303, + "epoch": 3.3093922651933703, + "grad_norm": 0.3219708800315857, + "learning_rate": 0.0002527814235597817, + "loss": 0.4262152099609375, + "mean_token_accuracy": 0.8510907486081123, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.479280876070261, + "epoch": 3.4321669736034375, + "grad_norm": 0.35597166419029236, + "learning_rate": 0.0002480839470819708, + "loss": 0.42484298706054685, + "mean_token_accuracy": 0.8504850694537163, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.48920651733875276, + "epoch": 3.554941682013505, + "grad_norm": 0.2936934232711792, + "learning_rate": 0.00024320573922507465, + "loss": 0.43512439727783203, + "mean_token_accuracy": 0.8497199699282646, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.47630224615335465, + "epoch": 3.677716390423573, + "grad_norm": 0.30153411626815796, + "learning_rate": 0.00023815572540539982, + "loss": 0.4236162567138672, + "mean_token_accuracy": 0.8507336723804474, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.4737357534468174, + "epoch": 3.80049109883364, + "grad_norm": 0.3035024106502533, + "learning_rate": 0.00023294314538414883, + "loss": 0.42273353576660155, + "mean_token_accuracy": 0.8513598147034646, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4796703179180622, + "epoch": 3.9232658072437077, + "grad_norm": 0.2834164500236511, + "learning_rate": 0.0002275775363618849, + "loss": 0.4326186752319336, + "mean_token_accuracy": 0.8495047062635421, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5010706964560917, + "eval_mean_token_accuracy": 0.8040556676047189, + "eval_not_syn_loss": 0.5851709842681885, + "eval_not_syn_runtime": 110.3987, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4733030595098223, + "eval_mean_token_accuracy": 0.8468134031976973, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5429526567459106, + "eval_syn_runtime": 117.3735, + "eval_syn_samples_per_second": 11.894, + "eval_syn_steps_per_second": 1.491, + "step": 1632 + }, + { + "entropy": 0.4337418030966357, + "epoch": 4.044198895027624, + "grad_norm": 0.31872114539146423, + "learning_rate": 0.00022206871552878668, + "loss": 0.3853663635253906, + "mean_token_accuracy": 0.8626761530256514, + "num_tokens": 4602893.0, + "step": 1650 + }, + { + "entropy": 0.39638415291905404, + "epoch": 4.166973603437691, + "grad_norm": 0.4238921105861664, + "learning_rate": 0.00021642676210261927, + "loss": 0.3409381866455078, + "mean_token_accuracy": 0.8753284150362015, + "num_tokens": 4735831.0, + "step": 1700 + }, + { + "entropy": 0.38611642628908155, + "epoch": 4.2897483118477595, + "grad_norm": 0.3248041868209839, + "learning_rate": 0.00021066199888728683, + "loss": 0.3360607147216797, + "mean_token_accuracy": 0.8773713061213493, + "num_tokens": 4880737.0, + "step": 1750 + }, + { + "entropy": 0.3938452216982842, + "epoch": 4.412523020257827, + "grad_norm": 0.33257895708084106, + "learning_rate": 0.00020478497338570733, + "loss": 0.34164436340332033, + "mean_token_accuracy": 0.8741997224092484, + "num_tokens": 5022337.0, + "step": 1800 + }, + { + "entropy": 0.39718300312757493, + "epoch": 4.535297728667894, + "grad_norm": 0.3596663177013397, + "learning_rate": 0.00019880643850156687, + "loss": 0.34481029510498046, + "mean_token_accuracy": 0.8744558349251748, + "num_tokens": 5158628.0, + "step": 1850 + }, + { + "entropy": 0.39785161226987836, + "epoch": 4.658072437077962, + "grad_norm": 0.3325240910053253, + "learning_rate": 0.00019273733286526186, + "loss": 0.34437469482421873, + "mean_token_accuracy": 0.8753779655694962, + "num_tokens": 5295686.0, + "step": 1900 + }, + { + "entropy": 0.3919269675016403, + "epoch": 4.780847145488029, + "grad_norm": 0.33487579226493835, + "learning_rate": 0.00018658876082002678, + "loss": 0.34184349060058594, + "mean_token_accuracy": 0.8743453392386437, + "num_tokens": 5440097.0, + "step": 1950 + }, + { + "entropy": 0.3975044973194599, + "epoch": 4.903621853898097, + "grad_norm": 0.27435103058815, + "learning_rate": 0.00018037197210486505, + "loss": 0.3441514205932617, + "mean_token_accuracy": 0.87376918643713, + "num_tokens": 5582367.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4433087486880166, + "eval_mean_token_accuracy": 0.8042705031803676, + "eval_not_syn_loss": 0.6243218183517456, + "eval_not_syn_runtime": 110.4378, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 5690965.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4196248413835253, + "eval_mean_token_accuracy": 0.8377946128164019, + "eval_num_tokens": 5690965.0, + "eval_syn_loss": 0.5790094137191772, + "eval_syn_runtime": 117.3572, + "eval_syn_samples_per_second": 11.895, + "eval_syn_steps_per_second": 1.491, + "step": 2040 + }, + { + "entropy": 0.3863225542954382, + "epoch": 5.024554941682013, + "grad_norm": 0.4834830164909363, + "learning_rate": 0.00017409834127145627, + "loss": 0.330703125, + "mean_token_accuracy": 0.8809086504926537, + "num_tokens": 5718747.0, + "step": 2050 + }, + { + "entropy": 0.3067230442166328, + "epoch": 5.147329650092081, + "grad_norm": 0.41984865069389343, + "learning_rate": 0.0001677793468727003, + "loss": 0.24890127182006835, + "mean_token_accuracy": 0.9052707189321518, + "num_tokens": 5862951.0, + "step": 2100 + }, + { + "entropy": 0.3144265574961901, + "epoch": 5.270104358502149, + "grad_norm": 0.3354702889919281, + "learning_rate": 0.00016142655046097487, + "loss": 0.25450078964233397, + "mean_token_accuracy": 0.9020006003975868, + "num_tokens": 6000342.0, + "step": 2150 + }, + { + "entropy": 0.3033023314923048, + "epoch": 5.392879066912216, + "grad_norm": 0.4292067289352417, + "learning_rate": 0.00015505157543453375, + "loss": 0.24809816360473633, + "mean_token_accuracy": 0.9048506420850754, + "num_tokens": 6145522.0, + "step": 2200 + }, + { + "entropy": 0.3102876263856888, + "epoch": 5.515653775322283, + "grad_norm": 0.47768503427505493, + "learning_rate": 0.00014866608577074797, + "loss": 0.2563666534423828, + "mean_token_accuracy": 0.901572678387165, + "num_tokens": 6282322.0, + "step": 2250 + }, + { + "entropy": 0.30644357711076736, + "epoch": 5.638428483732351, + "grad_norm": 0.4500775933265686, + "learning_rate": 0.00014228176468510215, + "loss": 0.2527992820739746, + "mean_token_accuracy": 0.9036021012067795, + "num_tokens": 6419205.0, + "step": 2300 + }, + { + "entropy": 0.31263820014894006, + "epoch": 5.7612031921424185, + "grad_norm": 0.36889463663101196, + "learning_rate": 0.00013591029325499087, + "loss": 0.2583304977416992, + "mean_token_accuracy": 0.9022675916552544, + "num_tokens": 6560369.0, + "step": 2350 + }, + { + "entropy": 0.31143927775323393, + "epoch": 5.883977900552486, + "grad_norm": 0.4023212790489197, + "learning_rate": 0.00012956332904742786, + "loss": 0.25584844589233396, + "mean_token_accuracy": 0.9025778490304946, + "num_tokens": 6700409.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.38324279410498485, + "eval_mean_token_accuracy": 0.8128126280648368, + "eval_not_syn_loss": 0.6938011646270752, + "eval_not_syn_runtime": 110.423, + "eval_not_syn_samples_per_second": 12.642, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 6829158.0, + "step": 2448 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3641860605989184, + "eval_mean_token_accuracy": 0.8323914204324995, + "eval_num_tokens": 6829158.0, + "eval_syn_loss": 0.6474246382713318, + "eval_syn_runtime": 117.3904, + "eval_syn_samples_per_second": 11.892, + "eval_syn_steps_per_second": 1.491, + "step": 2448 + }, + { + "entropy": 0.3088354336731325, + "epoch": 6.004910988336403, + "grad_norm": 0.2962503731250763, + "learning_rate": 0.0001232524847897706, + "loss": 0.2500633239746094, + "mean_token_accuracy": 0.9047863828349234, + "num_tokens": 6834835.0, + "step": 2450 + }, + { + "entropy": 0.2266372910887003, + "epoch": 6.12768569674647, + "grad_norm": 0.3523392379283905, + "learning_rate": 0.00011698930712248569, + "loss": 0.16580810546875, + "mean_token_accuracy": 0.9356516122817993, + "num_tokens": 6977951.0, + "step": 2500 + }, + { + "entropy": 0.22805758342146873, + "epoch": 6.250460405156538, + "grad_norm": 0.3685902953147888, + "learning_rate": 0.00011078525547283048, + "loss": 0.16887868881225587, + "mean_token_accuracy": 0.9341216534376144, + "num_tokens": 7121409.0, + "step": 2550 + }, + { + "entropy": 0.2327752672880888, + "epoch": 6.373235113566605, + "grad_norm": 0.4073072671890259, + "learning_rate": 0.00010465168108810399, + "loss": 0.17068164825439452, + "mean_token_accuracy": 0.9341617599129677, + "num_tokens": 7258294.0, + "step": 2600 + }, + { + "entropy": 0.23005703277885914, + "epoch": 6.496009821976672, + "grad_norm": 0.39206013083457947, + "learning_rate": 9.859980626682995e-05, + "loss": 0.1686391067504883, + "mean_token_accuracy": 0.9336268669366836, + "num_tokens": 7400869.0, + "step": 2650 + }, + { + "entropy": 0.22620332822203637, + "epoch": 6.6187845303867405, + "grad_norm": 0.3737471103668213, + "learning_rate": 9.26407038258705e-05, + "loss": 0.1691963005065918, + "mean_token_accuracy": 0.9336274519562722, + "num_tokens": 7537988.0, + "step": 2700 + }, + { + "entropy": 0.23150050349533557, + "epoch": 6.741559238796808, + "grad_norm": 0.4270040690898895, + "learning_rate": 8.678527684103929e-05, + "loss": 0.1724449348449707, + "mean_token_accuracy": 0.9331466138362885, + "num_tokens": 7674717.0, + "step": 2750 + }, + { + "entropy": 0.23078016005456448, + "epoch": 6.864333947206875, + "grad_norm": 0.40093865990638733, + "learning_rate": 8.104423869828156e-05, + "loss": 0.17107765197753907, + "mean_token_accuracy": 0.9330676954984665, + "num_tokens": 7809509.0, + "step": 2800 + }, + { + "entropy": 0.22516858130693435, + "epoch": 6.987108655616943, + "grad_norm": 0.40964797139167786, + "learning_rate": 7.542809349192022e-05, + "loss": 0.1664959144592285, + "mean_token_accuracy": 0.9353445410728455, + "num_tokens": 7953622.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.3169115138905389, + "eval_mean_token_accuracy": 0.8068022135325841, + "eval_not_syn_loss": 0.8101338148117065, + "eval_not_syn_runtime": 110.388, + "eval_not_syn_samples_per_second": 12.646, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 7967351.0, + "step": 2856 + }, + { + "epoch": 7.0, + "eval_entropy": 0.3000860639980861, + "eval_mean_token_accuracy": 0.8291359186172486, + "eval_num_tokens": 7967351.0, + "eval_syn_loss": 0.7574861645698547, + "eval_syn_runtime": 117.345, + "eval_syn_samples_per_second": 11.897, + "eval_syn_steps_per_second": 1.491, + "step": 2856 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2949283487417446e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9118eb8ff2c647ac8f1acb1826eb1cd73893fe31 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.06743035903930279, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0e4d664a0ad65173a331af923f9916044792f0f9 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3264/trainer_state.json @@ -0,0 +1,860 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3264, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.84347476541996, + "epoch": 0.12277470841006753, + "grad_norm": 0.5139632821083069, + "learning_rate": 3.586242144074487e-05, + "loss": 1.7810469055175782, + "mean_token_accuracy": 0.6252794374525547, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8905406814813613, + "epoch": 0.24554941682013506, + "grad_norm": 0.6561369299888611, + "learning_rate": 7.245672903334166e-05, + "loss": 0.83647705078125, + "mean_token_accuracy": 0.7502138108015061, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7635939045250416, + "epoch": 0.3683241252302026, + "grad_norm": 0.4626609683036804, + "learning_rate": 0.00010905103662593846, + "loss": 0.7156005096435547, + "mean_token_accuracy": 0.7786111453175545, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7165373960137367, + "epoch": 0.4910988336402701, + "grad_norm": 0.4098263680934906, + "learning_rate": 0.00014564534421853526, + "loss": 0.67410400390625, + "mean_token_accuracy": 0.7880693352222443, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7035001173615456, + "epoch": 0.6138735420503376, + "grad_norm": 0.33693116903305054, + "learning_rate": 0.0001822396518111321, + "loss": 0.6591246032714844, + "mean_token_accuracy": 0.7931166198849678, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6862310113012791, + "epoch": 0.7366482504604052, + "grad_norm": 0.31358352303504944, + "learning_rate": 0.00021883395940372884, + "loss": 0.6399201202392578, + "mean_token_accuracy": 0.7952846321463585, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6739526629447937, + "epoch": 0.8594229588704727, + "grad_norm": 0.4029310941696167, + "learning_rate": 0.00025542826699632564, + "loss": 0.6259587860107422, + "mean_token_accuracy": 0.8014272648096085, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6681017802655697, + "epoch": 0.9821976672805403, + "grad_norm": 0.3852674961090088, + "learning_rate": 0.00029202257458892247, + "loss": 0.6253739166259765, + "mean_token_accuracy": 0.8020894029736518, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6994867869785854, + "eval_mean_token_accuracy": 0.7849237663405282, + "eval_not_syn_loss": 0.6411319971084595, + "eval_not_syn_runtime": 110.7069, + "eval_not_syn_samples_per_second": 12.61, + "eval_not_syn_steps_per_second": 1.581, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6563669620241438, + "eval_mean_token_accuracy": 0.8207755245481219, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.6019997000694275, + "eval_syn_runtime": 117.5102, + "eval_syn_samples_per_second": 11.88, + "eval_syn_steps_per_second": 1.489, + "step": 408 + }, + { + "entropy": 0.6358311531809986, + "epoch": 1.1031307550644567, + "grad_norm": 0.34651753306388855, + "learning_rate": 0.00029851770373635705, + "loss": 0.5962887191772461, + "mean_token_accuracy": 0.8082216354796123, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6196171633899212, + "epoch": 1.2259054634745243, + "grad_norm": 0.28438520431518555, + "learning_rate": 0.0002981572760785325, + "loss": 0.5779812622070313, + "mean_token_accuracy": 0.8130937224626541, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.624696860164404, + "epoch": 1.3486801718845918, + "grad_norm": 0.3563199043273926, + "learning_rate": 0.000297524500341339, + "loss": 0.5760621643066406, + "mean_token_accuracy": 0.8123102071881294, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6383913996815681, + "epoch": 1.4714548802946594, + "grad_norm": 0.32306408882141113, + "learning_rate": 0.00029662053428333246, + "loss": 0.5874852371215821, + "mean_token_accuracy": 0.8098450502753258, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6132303845882415, + "epoch": 1.5942295887047269, + "grad_norm": 0.3351016044616699, + "learning_rate": 0.0002954470318466225, + "loss": 0.5624140930175782, + "mean_token_accuracy": 0.8152360209822654, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6104247760772705, + "epoch": 1.7170042971147943, + "grad_norm": 0.25214216113090515, + "learning_rate": 0.00029400614013073664, + "loss": 0.5608636093139648, + "mean_token_accuracy": 0.8154580116271972, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6093999670445919, + "epoch": 1.839779005524862, + "grad_norm": 0.25383421778678894, + "learning_rate": 0.0002923004954641779, + "loss": 0.5591114044189454, + "mean_token_accuracy": 0.817192807495594, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6029203486442566, + "epoch": 1.9625537139349294, + "grad_norm": 0.25239020586013794, + "learning_rate": 0.0002903332185808646, + "loss": 0.5529713821411133, + "mean_token_accuracy": 0.8182568901777267, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6090937239783151, + "eval_mean_token_accuracy": 0.7930077845709664, + "eval_not_syn_loss": 0.5995497107505798, + "eval_not_syn_runtime": 110.3965, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5718999467577253, + "eval_mean_token_accuracy": 0.84101799760546, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5510491728782654, + "eval_syn_runtime": 117.3523, + "eval_syn_samples_per_second": 11.896, + "eval_syn_steps_per_second": 1.491, + "step": 816 + }, + { + "entropy": 0.5504116184517817, + "epoch": 2.0834868017188457, + "grad_norm": 0.23869574069976807, + "learning_rate": 0.0002881079089102777, + "loss": 0.5012085723876953, + "mean_token_accuracy": 0.8299948473267144, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5487727333605289, + "epoch": 2.2062615101289134, + "grad_norm": 0.3087007403373718, + "learning_rate": 0.00028562863799176175, + "loss": 0.4988512802124023, + "mean_token_accuracy": 0.8319525212049484, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5354658082127571, + "epoch": 2.329036218538981, + "grad_norm": 0.2878682613372803, + "learning_rate": 0.00028289994202503066, + "loss": 0.4919636917114258, + "mean_token_accuracy": 0.8349191680550575, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5529813665151596, + "epoch": 2.4518109269490487, + "grad_norm": 0.2625614106655121, + "learning_rate": 0.00027992681357050643, + "loss": 0.5027856063842774, + "mean_token_accuracy": 0.8296491304039955, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5497122646868229, + "epoch": 2.574585635359116, + "grad_norm": 0.271993488073349, + "learning_rate": 0.00027671469241467785, + "loss": 0.49415691375732423, + "mean_token_accuracy": 0.8312779009342194, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5419583600759507, + "epoch": 2.6973603437691835, + "grad_norm": 0.23863548040390015, + "learning_rate": 0.00027326945561719136, + "loss": 0.49570159912109374, + "mean_token_accuracy": 0.8327477470040321, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5491574917733669, + "epoch": 2.820135052179251, + "grad_norm": 0.29003486037254333, + "learning_rate": 0.00026959740675788486, + "loss": 0.4965015411376953, + "mean_token_accuracy": 0.8336276519298553, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.5377901926636696, + "epoch": 2.942909760589319, + "grad_norm": 0.27723413705825806, + "learning_rate": 0.0002657052644034388, + "loss": 0.48964527130126956, + "mean_token_accuracy": 0.8356471425294876, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5576562706061772, + "eval_mean_token_accuracy": 0.817173547404153, + "eval_not_syn_loss": 0.5734513401985168, + "eval_not_syn_runtime": 110.4329, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5267024655001504, + "eval_mean_token_accuracy": 0.8285774500029428, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.544999361038208, + "eval_syn_runtime": 117.3289, + "eval_syn_samples_per_second": 11.898, + "eval_syn_steps_per_second": 1.492, + "step": 1224 + }, + { + "entropy": 0.502170312676938, + "epoch": 3.063842848373235, + "grad_norm": 0.44433876872062683, + "learning_rate": 0.0002616001498147458, + "loss": 0.448189811706543, + "mean_token_accuracy": 0.8443391725496592, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.46903550207614897, + "epoch": 3.1866175567833026, + "grad_norm": 0.2440977245569229, + "learning_rate": 0.0002572895739174909, + "loss": 0.4171265029907227, + "mean_token_accuracy": 0.8530975985527038, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4750825077295303, + "epoch": 3.3093922651933703, + "grad_norm": 0.3219708800315857, + "learning_rate": 0.0002527814235597817, + "loss": 0.4262152099609375, + "mean_token_accuracy": 0.8510907486081123, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.479280876070261, + "epoch": 3.4321669736034375, + "grad_norm": 0.35597166419029236, + "learning_rate": 0.0002480839470819708, + "loss": 0.42484298706054685, + "mean_token_accuracy": 0.8504850694537163, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.48920651733875276, + "epoch": 3.554941682013505, + "grad_norm": 0.2936934232711792, + "learning_rate": 0.00024320573922507465, + "loss": 0.43512439727783203, + "mean_token_accuracy": 0.8497199699282646, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.47630224615335465, + "epoch": 3.677716390423573, + "grad_norm": 0.30153411626815796, + "learning_rate": 0.00023815572540539982, + "loss": 0.4236162567138672, + "mean_token_accuracy": 0.8507336723804474, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.4737357534468174, + "epoch": 3.80049109883364, + "grad_norm": 0.3035024106502533, + "learning_rate": 0.00023294314538414883, + "loss": 0.42273353576660155, + "mean_token_accuracy": 0.8513598147034646, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4796703179180622, + "epoch": 3.9232658072437077, + "grad_norm": 0.2834164500236511, + "learning_rate": 0.0002275775363618849, + "loss": 0.4326186752319336, + "mean_token_accuracy": 0.8495047062635421, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5010706964560917, + "eval_mean_token_accuracy": 0.8040556676047189, + "eval_not_syn_loss": 0.5851709842681885, + "eval_not_syn_runtime": 110.3987, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4733030595098223, + "eval_mean_token_accuracy": 0.8468134031976973, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5429526567459106, + "eval_syn_runtime": 117.3735, + "eval_syn_samples_per_second": 11.894, + "eval_syn_steps_per_second": 1.491, + "step": 1632 + }, + { + "entropy": 0.4337418030966357, + "epoch": 4.044198895027624, + "grad_norm": 0.31872114539146423, + "learning_rate": 0.00022206871552878668, + "loss": 0.3853663635253906, + "mean_token_accuracy": 0.8626761530256514, + "num_tokens": 4602893.0, + "step": 1650 + }, + { + "entropy": 0.39638415291905404, + "epoch": 4.166973603437691, + "grad_norm": 0.4238921105861664, + "learning_rate": 0.00021642676210261927, + "loss": 0.3409381866455078, + "mean_token_accuracy": 0.8753284150362015, + "num_tokens": 4735831.0, + "step": 1700 + }, + { + "entropy": 0.38611642628908155, + "epoch": 4.2897483118477595, + "grad_norm": 0.3248041868209839, + "learning_rate": 0.00021066199888728683, + "loss": 0.3360607147216797, + "mean_token_accuracy": 0.8773713061213493, + "num_tokens": 4880737.0, + "step": 1750 + }, + { + "entropy": 0.3938452216982842, + "epoch": 4.412523020257827, + "grad_norm": 0.33257895708084106, + "learning_rate": 0.00020478497338570733, + "loss": 0.34164436340332033, + "mean_token_accuracy": 0.8741997224092484, + "num_tokens": 5022337.0, + "step": 1800 + }, + { + "entropy": 0.39718300312757493, + "epoch": 4.535297728667894, + "grad_norm": 0.3596663177013397, + "learning_rate": 0.00019880643850156687, + "loss": 0.34481029510498046, + "mean_token_accuracy": 0.8744558349251748, + "num_tokens": 5158628.0, + "step": 1850 + }, + { + "entropy": 0.39785161226987836, + "epoch": 4.658072437077962, + "grad_norm": 0.3325240910053253, + "learning_rate": 0.00019273733286526186, + "loss": 0.34437469482421873, + "mean_token_accuracy": 0.8753779655694962, + "num_tokens": 5295686.0, + "step": 1900 + }, + { + "entropy": 0.3919269675016403, + "epoch": 4.780847145488029, + "grad_norm": 0.33487579226493835, + "learning_rate": 0.00018658876082002678, + "loss": 0.34184349060058594, + "mean_token_accuracy": 0.8743453392386437, + "num_tokens": 5440097.0, + "step": 1950 + }, + { + "entropy": 0.3975044973194599, + "epoch": 4.903621853898097, + "grad_norm": 0.27435103058815, + "learning_rate": 0.00018037197210486505, + "loss": 0.3441514205932617, + "mean_token_accuracy": 0.87376918643713, + "num_tokens": 5582367.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4433087486880166, + "eval_mean_token_accuracy": 0.8042705031803676, + "eval_not_syn_loss": 0.6243218183517456, + "eval_not_syn_runtime": 110.4378, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 5690965.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4196248413835253, + "eval_mean_token_accuracy": 0.8377946128164019, + "eval_num_tokens": 5690965.0, + "eval_syn_loss": 0.5790094137191772, + "eval_syn_runtime": 117.3572, + "eval_syn_samples_per_second": 11.895, + "eval_syn_steps_per_second": 1.491, + "step": 2040 + }, + { + "entropy": 0.3863225542954382, + "epoch": 5.024554941682013, + "grad_norm": 0.4834830164909363, + "learning_rate": 0.00017409834127145627, + "loss": 0.330703125, + "mean_token_accuracy": 0.8809086504926537, + "num_tokens": 5718747.0, + "step": 2050 + }, + { + "entropy": 0.3067230442166328, + "epoch": 5.147329650092081, + "grad_norm": 0.41984865069389343, + "learning_rate": 0.0001677793468727003, + "loss": 0.24890127182006835, + "mean_token_accuracy": 0.9052707189321518, + "num_tokens": 5862951.0, + "step": 2100 + }, + { + "entropy": 0.3144265574961901, + "epoch": 5.270104358502149, + "grad_norm": 0.3354702889919281, + "learning_rate": 0.00016142655046097487, + "loss": 0.25450078964233397, + "mean_token_accuracy": 0.9020006003975868, + "num_tokens": 6000342.0, + "step": 2150 + }, + { + "entropy": 0.3033023314923048, + "epoch": 5.392879066912216, + "grad_norm": 0.4292067289352417, + "learning_rate": 0.00015505157543453375, + "loss": 0.24809816360473633, + "mean_token_accuracy": 0.9048506420850754, + "num_tokens": 6145522.0, + "step": 2200 + }, + { + "entropy": 0.3102876263856888, + "epoch": 5.515653775322283, + "grad_norm": 0.47768503427505493, + "learning_rate": 0.00014866608577074797, + "loss": 0.2563666534423828, + "mean_token_accuracy": 0.901572678387165, + "num_tokens": 6282322.0, + "step": 2250 + }, + { + "entropy": 0.30644357711076736, + "epoch": 5.638428483732351, + "grad_norm": 0.4500775933265686, + "learning_rate": 0.00014228176468510215, + "loss": 0.2527992820739746, + "mean_token_accuracy": 0.9036021012067795, + "num_tokens": 6419205.0, + "step": 2300 + }, + { + "entropy": 0.31263820014894006, + "epoch": 5.7612031921424185, + "grad_norm": 0.36889463663101196, + "learning_rate": 0.00013591029325499087, + "loss": 0.2583304977416992, + "mean_token_accuracy": 0.9022675916552544, + "num_tokens": 6560369.0, + "step": 2350 + }, + { + "entropy": 0.31143927775323393, + "epoch": 5.883977900552486, + "grad_norm": 0.4023212790489197, + "learning_rate": 0.00012956332904742786, + "loss": 0.25584844589233396, + "mean_token_accuracy": 0.9025778490304946, + "num_tokens": 6700409.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.38324279410498485, + "eval_mean_token_accuracy": 0.8128126280648368, + "eval_not_syn_loss": 0.6938011646270752, + "eval_not_syn_runtime": 110.423, + "eval_not_syn_samples_per_second": 12.642, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 6829158.0, + "step": 2448 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3641860605989184, + "eval_mean_token_accuracy": 0.8323914204324995, + "eval_num_tokens": 6829158.0, + "eval_syn_loss": 0.6474246382713318, + "eval_syn_runtime": 117.3904, + "eval_syn_samples_per_second": 11.892, + "eval_syn_steps_per_second": 1.491, + "step": 2448 + }, + { + "entropy": 0.3088354336731325, + "epoch": 6.004910988336403, + "grad_norm": 0.2962503731250763, + "learning_rate": 0.0001232524847897706, + "loss": 0.2500633239746094, + "mean_token_accuracy": 0.9047863828349234, + "num_tokens": 6834835.0, + "step": 2450 + }, + { + "entropy": 0.2266372910887003, + "epoch": 6.12768569674647, + "grad_norm": 0.3523392379283905, + "learning_rate": 0.00011698930712248569, + "loss": 0.16580810546875, + "mean_token_accuracy": 0.9356516122817993, + "num_tokens": 6977951.0, + "step": 2500 + }, + { + "entropy": 0.22805758342146873, + "epoch": 6.250460405156538, + "grad_norm": 0.3685902953147888, + "learning_rate": 0.00011078525547283048, + "loss": 0.16887868881225587, + "mean_token_accuracy": 0.9341216534376144, + "num_tokens": 7121409.0, + "step": 2550 + }, + { + "entropy": 0.2327752672880888, + "epoch": 6.373235113566605, + "grad_norm": 0.4073072671890259, + "learning_rate": 0.00010465168108810399, + "loss": 0.17068164825439452, + "mean_token_accuracy": 0.9341617599129677, + "num_tokens": 7258294.0, + "step": 2600 + }, + { + "entropy": 0.23005703277885914, + "epoch": 6.496009821976672, + "grad_norm": 0.39206013083457947, + "learning_rate": 9.859980626682995e-05, + "loss": 0.1686391067504883, + "mean_token_accuracy": 0.9336268669366836, + "num_tokens": 7400869.0, + "step": 2650 + }, + { + "entropy": 0.22620332822203637, + "epoch": 6.6187845303867405, + "grad_norm": 0.3737471103668213, + "learning_rate": 9.26407038258705e-05, + "loss": 0.1691963005065918, + "mean_token_accuracy": 0.9336274519562722, + "num_tokens": 7537988.0, + "step": 2700 + }, + { + "entropy": 0.23150050349533557, + "epoch": 6.741559238796808, + "grad_norm": 0.4270040690898895, + "learning_rate": 8.678527684103929e-05, + "loss": 0.1724449348449707, + "mean_token_accuracy": 0.9331466138362885, + "num_tokens": 7674717.0, + "step": 2750 + }, + { + "entropy": 0.23078016005456448, + "epoch": 6.864333947206875, + "grad_norm": 0.40093865990638733, + "learning_rate": 8.104423869828156e-05, + "loss": 0.17107765197753907, + "mean_token_accuracy": 0.9330676954984665, + "num_tokens": 7809509.0, + "step": 2800 + }, + { + "entropy": 0.22516858130693435, + "epoch": 6.987108655616943, + "grad_norm": 0.40964797139167786, + "learning_rate": 7.542809349192022e-05, + "loss": 0.1664959144592285, + "mean_token_accuracy": 0.9353445410728455, + "num_tokens": 7953622.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.3169115138905389, + "eval_mean_token_accuracy": 0.8068022135325841, + "eval_not_syn_loss": 0.8101338148117065, + "eval_not_syn_runtime": 110.388, + "eval_not_syn_samples_per_second": 12.646, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 7967351.0, + "step": 2856 + }, + { + "epoch": 7.0, + "eval_entropy": 0.3000860639980861, + "eval_mean_token_accuracy": 0.8291359186172486, + "eval_num_tokens": 7967351.0, + "eval_syn_loss": 0.7574861645698547, + "eval_syn_runtime": 117.345, + "eval_syn_samples_per_second": 11.897, + "eval_syn_steps_per_second": 1.491, + "step": 2856 + }, + { + "entropy": 0.18674430096058675, + "epoch": 7.108041743400859, + "grad_norm": 0.3196325898170471, + "learning_rate": 6.994711680583252e-05, + "loss": 0.12185008049011231, + "mean_token_accuracy": 0.953888118569621, + "num_tokens": 8083803.0, + "step": 2900 + }, + { + "entropy": 0.17206964828073978, + "epoch": 7.230816451810927, + "grad_norm": 0.3761785924434662, + "learning_rate": 6.461133691272128e-05, + "loss": 0.1051278018951416, + "mean_token_accuracy": 0.9581465849280357, + "num_tokens": 8226443.0, + "step": 2950 + }, + { + "entropy": 0.16801628757268192, + "epoch": 7.3535911602209945, + "grad_norm": 0.401041716337204, + "learning_rate": 5.943051642587869e-05, + "loss": 0.10529594421386719, + "mean_token_accuracy": 0.9593242672085762, + "num_tokens": 8368450.0, + "step": 3000 + }, + { + "entropy": 0.16075184397399425, + "epoch": 7.476365868631062, + "grad_norm": 0.45159032940864563, + "learning_rate": 5.441413443701439e-05, + "loss": 0.10433159828186035, + "mean_token_accuracy": 0.9596675795316696, + "num_tokens": 8514445.0, + "step": 3050 + }, + { + "entropy": 0.16641639694571495, + "epoch": 7.59914057704113, + "grad_norm": 0.439954936504364, + "learning_rate": 4.957136917282889e-05, + "loss": 0.10695881843566894, + "mean_token_accuracy": 0.9584988197684288, + "num_tokens": 8651914.0, + "step": 3100 + }, + { + "entropy": 0.1666870591044426, + "epoch": 7.721915285451197, + "grad_norm": 0.46230548620224, + "learning_rate": 4.4911081202065014e-05, + "loss": 0.10541806221008301, + "mean_token_accuracy": 0.9582247936725616, + "num_tokens": 8793102.0, + "step": 3150 + }, + { + "entropy": 0.16819031082093716, + "epoch": 7.844689993861264, + "grad_norm": 0.3943890333175659, + "learning_rate": 4.0441797223762166e-05, + "loss": 0.10882070541381836, + "mean_token_accuracy": 0.9577973237633706, + "num_tokens": 8930862.0, + "step": 3200 + }, + { + "entropy": 0.16455162566155196, + "epoch": 7.967464702271332, + "grad_norm": 0.4292966425418854, + "learning_rate": 3.617169446637561e-05, + "loss": 0.10696003913879394, + "mean_token_accuracy": 0.9589823609590531, + "num_tokens": 9068863.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.27482904161725724, + "eval_mean_token_accuracy": 0.8073075127601623, + "eval_not_syn_loss": 0.936057448387146, + "eval_not_syn_runtime": 110.4123, + "eval_not_syn_samples_per_second": 12.644, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 9105544.0, + "step": 3264 + }, + { + "epoch": 8.0, + "eval_entropy": 0.25925376977239334, + "eval_mean_token_accuracy": 0.8265137771197728, + "eval_num_tokens": 9105544.0, + "eval_syn_loss": 0.882119357585907, + "eval_syn_runtime": 117.3341, + "eval_syn_samples_per_second": 11.898, + "eval_syn_steps_per_second": 1.491, + "step": 3264 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.479619450859182e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9118eb8ff2c647ac8f1acb1826eb1cd73893fe31 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.06743035903930279, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a4589277bca3b29a5f10241ad6ec2ed6051ed766 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-3672/trainer_state.json @@ -0,0 +1,962 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3672, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.84347476541996, + "epoch": 0.12277470841006753, + "grad_norm": 0.5139632821083069, + "learning_rate": 3.586242144074487e-05, + "loss": 1.7810469055175782, + "mean_token_accuracy": 0.6252794374525547, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8905406814813613, + "epoch": 0.24554941682013506, + "grad_norm": 0.6561369299888611, + "learning_rate": 7.245672903334166e-05, + "loss": 0.83647705078125, + "mean_token_accuracy": 0.7502138108015061, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7635939045250416, + "epoch": 0.3683241252302026, + "grad_norm": 0.4626609683036804, + "learning_rate": 0.00010905103662593846, + "loss": 0.7156005096435547, + "mean_token_accuracy": 0.7786111453175545, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7165373960137367, + "epoch": 0.4910988336402701, + "grad_norm": 0.4098263680934906, + "learning_rate": 0.00014564534421853526, + "loss": 0.67410400390625, + "mean_token_accuracy": 0.7880693352222443, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7035001173615456, + "epoch": 0.6138735420503376, + "grad_norm": 0.33693116903305054, + "learning_rate": 0.0001822396518111321, + "loss": 0.6591246032714844, + "mean_token_accuracy": 0.7931166198849678, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6862310113012791, + "epoch": 0.7366482504604052, + "grad_norm": 0.31358352303504944, + "learning_rate": 0.00021883395940372884, + "loss": 0.6399201202392578, + "mean_token_accuracy": 0.7952846321463585, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6739526629447937, + "epoch": 0.8594229588704727, + "grad_norm": 0.4029310941696167, + "learning_rate": 0.00025542826699632564, + "loss": 0.6259587860107422, + "mean_token_accuracy": 0.8014272648096085, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6681017802655697, + "epoch": 0.9821976672805403, + "grad_norm": 0.3852674961090088, + "learning_rate": 0.00029202257458892247, + "loss": 0.6253739166259765, + "mean_token_accuracy": 0.8020894029736518, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6994867869785854, + "eval_mean_token_accuracy": 0.7849237663405282, + "eval_not_syn_loss": 0.6411319971084595, + "eval_not_syn_runtime": 110.7069, + "eval_not_syn_samples_per_second": 12.61, + "eval_not_syn_steps_per_second": 1.581, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6563669620241438, + "eval_mean_token_accuracy": 0.8207755245481219, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.6019997000694275, + "eval_syn_runtime": 117.5102, + "eval_syn_samples_per_second": 11.88, + "eval_syn_steps_per_second": 1.489, + "step": 408 + }, + { + "entropy": 0.6358311531809986, + "epoch": 1.1031307550644567, + "grad_norm": 0.34651753306388855, + "learning_rate": 0.00029851770373635705, + "loss": 0.5962887191772461, + "mean_token_accuracy": 0.8082216354796123, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6196171633899212, + "epoch": 1.2259054634745243, + "grad_norm": 0.28438520431518555, + "learning_rate": 0.0002981572760785325, + "loss": 0.5779812622070313, + "mean_token_accuracy": 0.8130937224626541, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.624696860164404, + "epoch": 1.3486801718845918, + "grad_norm": 0.3563199043273926, + "learning_rate": 0.000297524500341339, + "loss": 0.5760621643066406, + "mean_token_accuracy": 0.8123102071881294, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6383913996815681, + "epoch": 1.4714548802946594, + "grad_norm": 0.32306408882141113, + "learning_rate": 0.00029662053428333246, + "loss": 0.5874852371215821, + "mean_token_accuracy": 0.8098450502753258, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6132303845882415, + "epoch": 1.5942295887047269, + "grad_norm": 0.3351016044616699, + "learning_rate": 0.0002954470318466225, + "loss": 0.5624140930175782, + "mean_token_accuracy": 0.8152360209822654, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6104247760772705, + "epoch": 1.7170042971147943, + "grad_norm": 0.25214216113090515, + "learning_rate": 0.00029400614013073664, + "loss": 0.5608636093139648, + "mean_token_accuracy": 0.8154580116271972, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6093999670445919, + "epoch": 1.839779005524862, + "grad_norm": 0.25383421778678894, + "learning_rate": 0.0002923004954641779, + "loss": 0.5591114044189454, + "mean_token_accuracy": 0.817192807495594, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6029203486442566, + "epoch": 1.9625537139349294, + "grad_norm": 0.25239020586013794, + "learning_rate": 0.0002903332185808646, + "loss": 0.5529713821411133, + "mean_token_accuracy": 0.8182568901777267, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6090937239783151, + "eval_mean_token_accuracy": 0.7930077845709664, + "eval_not_syn_loss": 0.5995497107505798, + "eval_not_syn_runtime": 110.3965, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5718999467577253, + "eval_mean_token_accuracy": 0.84101799760546, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5510491728782654, + "eval_syn_runtime": 117.3523, + "eval_syn_samples_per_second": 11.896, + "eval_syn_steps_per_second": 1.491, + "step": 816 + }, + { + "entropy": 0.5504116184517817, + "epoch": 2.0834868017188457, + "grad_norm": 0.23869574069976807, + "learning_rate": 0.0002881079089102777, + "loss": 0.5012085723876953, + "mean_token_accuracy": 0.8299948473267144, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5487727333605289, + "epoch": 2.2062615101289134, + "grad_norm": 0.3087007403373718, + "learning_rate": 0.00028562863799176175, + "loss": 0.4988512802124023, + "mean_token_accuracy": 0.8319525212049484, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5354658082127571, + "epoch": 2.329036218538981, + "grad_norm": 0.2878682613372803, + "learning_rate": 0.00028289994202503066, + "loss": 0.4919636917114258, + "mean_token_accuracy": 0.8349191680550575, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5529813665151596, + "epoch": 2.4518109269490487, + "grad_norm": 0.2625614106655121, + "learning_rate": 0.00027992681357050643, + "loss": 0.5027856063842774, + "mean_token_accuracy": 0.8296491304039955, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5497122646868229, + "epoch": 2.574585635359116, + "grad_norm": 0.271993488073349, + "learning_rate": 0.00027671469241467785, + "loss": 0.49415691375732423, + "mean_token_accuracy": 0.8312779009342194, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5419583600759507, + "epoch": 2.6973603437691835, + "grad_norm": 0.23863548040390015, + "learning_rate": 0.00027326945561719136, + "loss": 0.49570159912109374, + "mean_token_accuracy": 0.8327477470040321, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5491574917733669, + "epoch": 2.820135052179251, + "grad_norm": 0.29003486037254333, + "learning_rate": 0.00026959740675788486, + "loss": 0.4965015411376953, + "mean_token_accuracy": 0.8336276519298553, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.5377901926636696, + "epoch": 2.942909760589319, + "grad_norm": 0.27723413705825806, + "learning_rate": 0.0002657052644034388, + "loss": 0.48964527130126956, + "mean_token_accuracy": 0.8356471425294876, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5576562706061772, + "eval_mean_token_accuracy": 0.817173547404153, + "eval_not_syn_loss": 0.5734513401985168, + "eval_not_syn_runtime": 110.4329, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5267024655001504, + "eval_mean_token_accuracy": 0.8285774500029428, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.544999361038208, + "eval_syn_runtime": 117.3289, + "eval_syn_samples_per_second": 11.898, + "eval_syn_steps_per_second": 1.492, + "step": 1224 + }, + { + "entropy": 0.502170312676938, + "epoch": 3.063842848373235, + "grad_norm": 0.44433876872062683, + "learning_rate": 0.0002616001498147458, + "loss": 0.448189811706543, + "mean_token_accuracy": 0.8443391725496592, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.46903550207614897, + "epoch": 3.1866175567833026, + "grad_norm": 0.2440977245569229, + "learning_rate": 0.0002572895739174909, + "loss": 0.4171265029907227, + "mean_token_accuracy": 0.8530975985527038, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4750825077295303, + "epoch": 3.3093922651933703, + "grad_norm": 0.3219708800315857, + "learning_rate": 0.0002527814235597817, + "loss": 0.4262152099609375, + "mean_token_accuracy": 0.8510907486081123, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.479280876070261, + "epoch": 3.4321669736034375, + "grad_norm": 0.35597166419029236, + "learning_rate": 0.0002480839470819708, + "loss": 0.42484298706054685, + "mean_token_accuracy": 0.8504850694537163, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.48920651733875276, + "epoch": 3.554941682013505, + "grad_norm": 0.2936934232711792, + "learning_rate": 0.00024320573922507465, + "loss": 0.43512439727783203, + "mean_token_accuracy": 0.8497199699282646, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.47630224615335465, + "epoch": 3.677716390423573, + "grad_norm": 0.30153411626815796, + "learning_rate": 0.00023815572540539982, + "loss": 0.4236162567138672, + "mean_token_accuracy": 0.8507336723804474, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.4737357534468174, + "epoch": 3.80049109883364, + "grad_norm": 0.3035024106502533, + "learning_rate": 0.00023294314538414883, + "loss": 0.42273353576660155, + "mean_token_accuracy": 0.8513598147034646, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4796703179180622, + "epoch": 3.9232658072437077, + "grad_norm": 0.2834164500236511, + "learning_rate": 0.0002275775363618849, + "loss": 0.4326186752319336, + "mean_token_accuracy": 0.8495047062635421, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5010706964560917, + "eval_mean_token_accuracy": 0.8040556676047189, + "eval_not_syn_loss": 0.5851709842681885, + "eval_not_syn_runtime": 110.3987, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4733030595098223, + "eval_mean_token_accuracy": 0.8468134031976973, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5429526567459106, + "eval_syn_runtime": 117.3735, + "eval_syn_samples_per_second": 11.894, + "eval_syn_steps_per_second": 1.491, + "step": 1632 + }, + { + "entropy": 0.4337418030966357, + "epoch": 4.044198895027624, + "grad_norm": 0.31872114539146423, + "learning_rate": 0.00022206871552878668, + "loss": 0.3853663635253906, + "mean_token_accuracy": 0.8626761530256514, + "num_tokens": 4602893.0, + "step": 1650 + }, + { + "entropy": 0.39638415291905404, + "epoch": 4.166973603437691, + "grad_norm": 0.4238921105861664, + "learning_rate": 0.00021642676210261927, + "loss": 0.3409381866455078, + "mean_token_accuracy": 0.8753284150362015, + "num_tokens": 4735831.0, + "step": 1700 + }, + { + "entropy": 0.38611642628908155, + "epoch": 4.2897483118477595, + "grad_norm": 0.3248041868209839, + "learning_rate": 0.00021066199888728683, + "loss": 0.3360607147216797, + "mean_token_accuracy": 0.8773713061213493, + "num_tokens": 4880737.0, + "step": 1750 + }, + { + "entropy": 0.3938452216982842, + "epoch": 4.412523020257827, + "grad_norm": 0.33257895708084106, + "learning_rate": 0.00020478497338570733, + "loss": 0.34164436340332033, + "mean_token_accuracy": 0.8741997224092484, + "num_tokens": 5022337.0, + "step": 1800 + }, + { + "entropy": 0.39718300312757493, + "epoch": 4.535297728667894, + "grad_norm": 0.3596663177013397, + "learning_rate": 0.00019880643850156687, + "loss": 0.34481029510498046, + "mean_token_accuracy": 0.8744558349251748, + "num_tokens": 5158628.0, + "step": 1850 + }, + { + "entropy": 0.39785161226987836, + "epoch": 4.658072437077962, + "grad_norm": 0.3325240910053253, + "learning_rate": 0.00019273733286526186, + "loss": 0.34437469482421873, + "mean_token_accuracy": 0.8753779655694962, + "num_tokens": 5295686.0, + "step": 1900 + }, + { + "entropy": 0.3919269675016403, + "epoch": 4.780847145488029, + "grad_norm": 0.33487579226493835, + "learning_rate": 0.00018658876082002678, + "loss": 0.34184349060058594, + "mean_token_accuracy": 0.8743453392386437, + "num_tokens": 5440097.0, + "step": 1950 + }, + { + "entropy": 0.3975044973194599, + "epoch": 4.903621853898097, + "grad_norm": 0.27435103058815, + "learning_rate": 0.00018037197210486505, + "loss": 0.3441514205932617, + "mean_token_accuracy": 0.87376918643713, + "num_tokens": 5582367.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4433087486880166, + "eval_mean_token_accuracy": 0.8042705031803676, + "eval_not_syn_loss": 0.6243218183517456, + "eval_not_syn_runtime": 110.4378, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 5690965.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4196248413835253, + "eval_mean_token_accuracy": 0.8377946128164019, + "eval_num_tokens": 5690965.0, + "eval_syn_loss": 0.5790094137191772, + "eval_syn_runtime": 117.3572, + "eval_syn_samples_per_second": 11.895, + "eval_syn_steps_per_second": 1.491, + "step": 2040 + }, + { + "entropy": 0.3863225542954382, + "epoch": 5.024554941682013, + "grad_norm": 0.4834830164909363, + "learning_rate": 0.00017409834127145627, + "loss": 0.330703125, + "mean_token_accuracy": 0.8809086504926537, + "num_tokens": 5718747.0, + "step": 2050 + }, + { + "entropy": 0.3067230442166328, + "epoch": 5.147329650092081, + "grad_norm": 0.41984865069389343, + "learning_rate": 0.0001677793468727003, + "loss": 0.24890127182006835, + "mean_token_accuracy": 0.9052707189321518, + "num_tokens": 5862951.0, + "step": 2100 + }, + { + "entropy": 0.3144265574961901, + "epoch": 5.270104358502149, + "grad_norm": 0.3354702889919281, + "learning_rate": 0.00016142655046097487, + "loss": 0.25450078964233397, + "mean_token_accuracy": 0.9020006003975868, + "num_tokens": 6000342.0, + "step": 2150 + }, + { + "entropy": 0.3033023314923048, + "epoch": 5.392879066912216, + "grad_norm": 0.4292067289352417, + "learning_rate": 0.00015505157543453375, + "loss": 0.24809816360473633, + "mean_token_accuracy": 0.9048506420850754, + "num_tokens": 6145522.0, + "step": 2200 + }, + { + "entropy": 0.3102876263856888, + "epoch": 5.515653775322283, + "grad_norm": 0.47768503427505493, + "learning_rate": 0.00014866608577074797, + "loss": 0.2563666534423828, + "mean_token_accuracy": 0.901572678387165, + "num_tokens": 6282322.0, + "step": 2250 + }, + { + "entropy": 0.30644357711076736, + "epoch": 5.638428483732351, + "grad_norm": 0.4500775933265686, + "learning_rate": 0.00014228176468510215, + "loss": 0.2527992820739746, + "mean_token_accuracy": 0.9036021012067795, + "num_tokens": 6419205.0, + "step": 2300 + }, + { + "entropy": 0.31263820014894006, + "epoch": 5.7612031921424185, + "grad_norm": 0.36889463663101196, + "learning_rate": 0.00013591029325499087, + "loss": 0.2583304977416992, + "mean_token_accuracy": 0.9022675916552544, + "num_tokens": 6560369.0, + "step": 2350 + }, + { + "entropy": 0.31143927775323393, + "epoch": 5.883977900552486, + "grad_norm": 0.4023212790489197, + "learning_rate": 0.00012956332904742786, + "loss": 0.25584844589233396, + "mean_token_accuracy": 0.9025778490304946, + "num_tokens": 6700409.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.38324279410498485, + "eval_mean_token_accuracy": 0.8128126280648368, + "eval_not_syn_loss": 0.6938011646270752, + "eval_not_syn_runtime": 110.423, + "eval_not_syn_samples_per_second": 12.642, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 6829158.0, + "step": 2448 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3641860605989184, + "eval_mean_token_accuracy": 0.8323914204324995, + "eval_num_tokens": 6829158.0, + "eval_syn_loss": 0.6474246382713318, + "eval_syn_runtime": 117.3904, + "eval_syn_samples_per_second": 11.892, + "eval_syn_steps_per_second": 1.491, + "step": 2448 + }, + { + "entropy": 0.3088354336731325, + "epoch": 6.004910988336403, + "grad_norm": 0.2962503731250763, + "learning_rate": 0.0001232524847897706, + "loss": 0.2500633239746094, + "mean_token_accuracy": 0.9047863828349234, + "num_tokens": 6834835.0, + "step": 2450 + }, + { + "entropy": 0.2266372910887003, + "epoch": 6.12768569674647, + "grad_norm": 0.3523392379283905, + "learning_rate": 0.00011698930712248569, + "loss": 0.16580810546875, + "mean_token_accuracy": 0.9356516122817993, + "num_tokens": 6977951.0, + "step": 2500 + }, + { + "entropy": 0.22805758342146873, + "epoch": 6.250460405156538, + "grad_norm": 0.3685902953147888, + "learning_rate": 0.00011078525547283048, + "loss": 0.16887868881225587, + "mean_token_accuracy": 0.9341216534376144, + "num_tokens": 7121409.0, + "step": 2550 + }, + { + "entropy": 0.2327752672880888, + "epoch": 6.373235113566605, + "grad_norm": 0.4073072671890259, + "learning_rate": 0.00010465168108810399, + "loss": 0.17068164825439452, + "mean_token_accuracy": 0.9341617599129677, + "num_tokens": 7258294.0, + "step": 2600 + }, + { + "entropy": 0.23005703277885914, + "epoch": 6.496009821976672, + "grad_norm": 0.39206013083457947, + "learning_rate": 9.859980626682995e-05, + "loss": 0.1686391067504883, + "mean_token_accuracy": 0.9336268669366836, + "num_tokens": 7400869.0, + "step": 2650 + }, + { + "entropy": 0.22620332822203637, + "epoch": 6.6187845303867405, + "grad_norm": 0.3737471103668213, + "learning_rate": 9.26407038258705e-05, + "loss": 0.1691963005065918, + "mean_token_accuracy": 0.9336274519562722, + "num_tokens": 7537988.0, + "step": 2700 + }, + { + "entropy": 0.23150050349533557, + "epoch": 6.741559238796808, + "grad_norm": 0.4270040690898895, + "learning_rate": 8.678527684103929e-05, + "loss": 0.1724449348449707, + "mean_token_accuracy": 0.9331466138362885, + "num_tokens": 7674717.0, + "step": 2750 + }, + { + "entropy": 0.23078016005456448, + "epoch": 6.864333947206875, + "grad_norm": 0.40093865990638733, + "learning_rate": 8.104423869828156e-05, + "loss": 0.17107765197753907, + "mean_token_accuracy": 0.9330676954984665, + "num_tokens": 7809509.0, + "step": 2800 + }, + { + "entropy": 0.22516858130693435, + "epoch": 6.987108655616943, + "grad_norm": 0.40964797139167786, + "learning_rate": 7.542809349192022e-05, + "loss": 0.1664959144592285, + "mean_token_accuracy": 0.9353445410728455, + "num_tokens": 7953622.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.3169115138905389, + "eval_mean_token_accuracy": 0.8068022135325841, + "eval_not_syn_loss": 0.8101338148117065, + "eval_not_syn_runtime": 110.388, + "eval_not_syn_samples_per_second": 12.646, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 7967351.0, + "step": 2856 + }, + { + "epoch": 7.0, + "eval_entropy": 0.3000860639980861, + "eval_mean_token_accuracy": 0.8291359186172486, + "eval_num_tokens": 7967351.0, + "eval_syn_loss": 0.7574861645698547, + "eval_syn_runtime": 117.345, + "eval_syn_samples_per_second": 11.897, + "eval_syn_steps_per_second": 1.491, + "step": 2856 + }, + { + "entropy": 0.18674430096058675, + "epoch": 7.108041743400859, + "grad_norm": 0.3196325898170471, + "learning_rate": 6.994711680583252e-05, + "loss": 0.12185008049011231, + "mean_token_accuracy": 0.953888118569621, + "num_tokens": 8083803.0, + "step": 2900 + }, + { + "entropy": 0.17206964828073978, + "epoch": 7.230816451810927, + "grad_norm": 0.3761785924434662, + "learning_rate": 6.461133691272128e-05, + "loss": 0.1051278018951416, + "mean_token_accuracy": 0.9581465849280357, + "num_tokens": 8226443.0, + "step": 2950 + }, + { + "entropy": 0.16801628757268192, + "epoch": 7.3535911602209945, + "grad_norm": 0.401041716337204, + "learning_rate": 5.943051642587869e-05, + "loss": 0.10529594421386719, + "mean_token_accuracy": 0.9593242672085762, + "num_tokens": 8368450.0, + "step": 3000 + }, + { + "entropy": 0.16075184397399425, + "epoch": 7.476365868631062, + "grad_norm": 0.45159032940864563, + "learning_rate": 5.441413443701439e-05, + "loss": 0.10433159828186035, + "mean_token_accuracy": 0.9596675795316696, + "num_tokens": 8514445.0, + "step": 3050 + }, + { + "entropy": 0.16641639694571495, + "epoch": 7.59914057704113, + "grad_norm": 0.439954936504364, + "learning_rate": 4.957136917282889e-05, + "loss": 0.10695881843566894, + "mean_token_accuracy": 0.9584988197684288, + "num_tokens": 8651914.0, + "step": 3100 + }, + { + "entropy": 0.1666870591044426, + "epoch": 7.721915285451197, + "grad_norm": 0.46230548620224, + "learning_rate": 4.4911081202065014e-05, + "loss": 0.10541806221008301, + "mean_token_accuracy": 0.9582247936725616, + "num_tokens": 8793102.0, + "step": 3150 + }, + { + "entropy": 0.16819031082093716, + "epoch": 7.844689993861264, + "grad_norm": 0.3943890333175659, + "learning_rate": 4.0441797223762166e-05, + "loss": 0.10882070541381836, + "mean_token_accuracy": 0.9577973237633706, + "num_tokens": 8930862.0, + "step": 3200 + }, + { + "entropy": 0.16455162566155196, + "epoch": 7.967464702271332, + "grad_norm": 0.4292966425418854, + "learning_rate": 3.617169446637561e-05, + "loss": 0.10696003913879394, + "mean_token_accuracy": 0.9589823609590531, + "num_tokens": 9068863.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.27482904161725724, + "eval_mean_token_accuracy": 0.8073075127601623, + "eval_not_syn_loss": 0.936057448387146, + "eval_not_syn_runtime": 110.4123, + "eval_not_syn_samples_per_second": 12.644, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 9105544.0, + "step": 3264 + }, + { + "epoch": 8.0, + "eval_entropy": 0.25925376977239334, + "eval_mean_token_accuracy": 0.8265137771197728, + "eval_num_tokens": 9105544.0, + "eval_syn_loss": 0.882119357585907, + "eval_syn_runtime": 117.3341, + "eval_syn_samples_per_second": 11.898, + "eval_syn_steps_per_second": 1.491, + "step": 3264 + }, + { + "entropy": 0.14927380497988105, + "epoch": 8.088397790055248, + "grad_norm": 0.2612760066986084, + "learning_rate": 3.210858572630498e-05, + "loss": 0.08452132225036621, + "mean_token_accuracy": 0.9678469657292826, + "num_tokens": 9202212.0, + "step": 3300 + }, + { + "entropy": 0.13725149285048246, + "epoch": 8.211172498465316, + "grad_norm": 0.23544830083847046, + "learning_rate": 2.8259905073206182e-05, + "loss": 0.07458745956420898, + "mean_token_accuracy": 0.9708940279483795, + "num_tokens": 9340486.0, + "step": 3350 + }, + { + "entropy": 0.1391688558831811, + "epoch": 8.333947206875383, + "grad_norm": 0.23687265813350677, + "learning_rate": 2.4632694248240548e-05, + "loss": 0.07527658462524414, + "mean_token_accuracy": 0.9697121676802635, + "num_tokens": 9476614.0, + "step": 3400 + }, + { + "entropy": 0.1378477082774043, + "epoch": 8.456721915285451, + "grad_norm": 0.4248947501182556, + "learning_rate": 2.123358978014849e-05, + "loss": 0.07582931995391845, + "mean_token_accuracy": 0.9704358732700348, + "num_tokens": 9614420.0, + "step": 3450 + }, + { + "entropy": 0.12935656122863293, + "epoch": 8.579496623695519, + "grad_norm": 0.2261466085910797, + "learning_rate": 1.8068810842720363e-05, + "loss": 0.07171549797058105, + "mean_token_accuracy": 0.9720115476846695, + "num_tokens": 9764329.0, + "step": 3500 + }, + { + "entropy": 0.137032612003386, + "epoch": 8.702271332105585, + "grad_norm": 0.28648674488067627, + "learning_rate": 1.5144147875880633e-05, + "loss": 0.07382936477661133, + "mean_token_accuracy": 0.9700555896759033, + "num_tokens": 9907026.0, + "step": 3550 + }, + { + "entropy": 0.13177497897297144, + "epoch": 8.825046040515653, + "grad_norm": 0.2325354367494583, + "learning_rate": 1.2464951991205664e-05, + "loss": 0.07414592742919922, + "mean_token_accuracy": 0.9710717841982841, + "num_tokens": 10048663.0, + "step": 3600 + }, + { + "entropy": 0.13154912155121565, + "epoch": 8.947820748925722, + "grad_norm": 0.316993772983551, + "learning_rate": 1.0036125181258958e-05, + "loss": 0.07341423034667968, + "mean_token_accuracy": 0.9714728254079819, + "num_tokens": 10188058.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.24302378271307265, + "eval_mean_token_accuracy": 0.8097811685289655, + "eval_not_syn_loss": 1.0759197473526, + "eval_not_syn_runtime": 110.4176, + "eval_not_syn_samples_per_second": 12.643, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 10243737.0, + "step": 3672 + }, + { + "epoch": 9.0, + "eval_entropy": 0.22846667877265386, + "eval_mean_token_accuracy": 0.8239966991969517, + "eval_num_tokens": 10243737.0, + "eval_syn_loss": 1.0279099941253662, + "eval_syn_runtime": 117.3139, + "eval_syn_samples_per_second": 11.9, + "eval_syn_steps_per_second": 1.492, + "step": 3672 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6641560363321242e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9118eb8ff2c647ac8f1acb1826eb1cd73893fe31 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.06743035903930279, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4967e8278b339dd0c1a759cf589d57b5818f17c7 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-408/trainer_state.json @@ -0,0 +1,136 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 408, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.84347476541996, + "epoch": 0.12277470841006753, + "grad_norm": 0.5139632821083069, + "learning_rate": 3.586242144074487e-05, + "loss": 1.7810469055175782, + "mean_token_accuracy": 0.6252794374525547, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8905406814813613, + "epoch": 0.24554941682013506, + "grad_norm": 0.6561369299888611, + "learning_rate": 7.245672903334166e-05, + "loss": 0.83647705078125, + "mean_token_accuracy": 0.7502138108015061, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7635939045250416, + "epoch": 0.3683241252302026, + "grad_norm": 0.4626609683036804, + "learning_rate": 0.00010905103662593846, + "loss": 0.7156005096435547, + "mean_token_accuracy": 0.7786111453175545, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7165373960137367, + "epoch": 0.4910988336402701, + "grad_norm": 0.4098263680934906, + "learning_rate": 0.00014564534421853526, + "loss": 0.67410400390625, + "mean_token_accuracy": 0.7880693352222443, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7035001173615456, + "epoch": 0.6138735420503376, + "grad_norm": 0.33693116903305054, + "learning_rate": 0.0001822396518111321, + "loss": 0.6591246032714844, + "mean_token_accuracy": 0.7931166198849678, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6862310113012791, + "epoch": 0.7366482504604052, + "grad_norm": 0.31358352303504944, + "learning_rate": 0.00021883395940372884, + "loss": 0.6399201202392578, + "mean_token_accuracy": 0.7952846321463585, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6739526629447937, + "epoch": 0.8594229588704727, + "grad_norm": 0.4029310941696167, + "learning_rate": 0.00025542826699632564, + "loss": 0.6259587860107422, + "mean_token_accuracy": 0.8014272648096085, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6681017802655697, + "epoch": 0.9821976672805403, + "grad_norm": 0.3852674961090088, + "learning_rate": 0.00029202257458892247, + "loss": 0.6253739166259765, + "mean_token_accuracy": 0.8020894029736518, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6994867869785854, + "eval_mean_token_accuracy": 0.7849237663405282, + "eval_not_syn_loss": 0.6411319971084595, + "eval_not_syn_runtime": 110.7069, + "eval_not_syn_samples_per_second": 12.61, + "eval_not_syn_steps_per_second": 1.581, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6563669620241438, + "eval_mean_token_accuracy": 0.8207755245481219, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.6019997000694275, + "eval_syn_runtime": 117.5102, + "eval_syn_samples_per_second": 11.88, + "eval_syn_steps_per_second": 1.489, + "step": 408 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.8530281521497088e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9118eb8ff2c647ac8f1acb1826eb1cd73893fe31 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.06743035903930279, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8d17499a1a7d41270757cc93216c18bdf528a820 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-4080/trainer_state.json @@ -0,0 +1,1064 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4080, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.84347476541996, + "epoch": 0.12277470841006753, + "grad_norm": 0.5139632821083069, + "learning_rate": 3.586242144074487e-05, + "loss": 1.7810469055175782, + "mean_token_accuracy": 0.6252794374525547, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8905406814813613, + "epoch": 0.24554941682013506, + "grad_norm": 0.6561369299888611, + "learning_rate": 7.245672903334166e-05, + "loss": 0.83647705078125, + "mean_token_accuracy": 0.7502138108015061, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7635939045250416, + "epoch": 0.3683241252302026, + "grad_norm": 0.4626609683036804, + "learning_rate": 0.00010905103662593846, + "loss": 0.7156005096435547, + "mean_token_accuracy": 0.7786111453175545, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7165373960137367, + "epoch": 0.4910988336402701, + "grad_norm": 0.4098263680934906, + "learning_rate": 0.00014564534421853526, + "loss": 0.67410400390625, + "mean_token_accuracy": 0.7880693352222443, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7035001173615456, + "epoch": 0.6138735420503376, + "grad_norm": 0.33693116903305054, + "learning_rate": 0.0001822396518111321, + "loss": 0.6591246032714844, + "mean_token_accuracy": 0.7931166198849678, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6862310113012791, + "epoch": 0.7366482504604052, + "grad_norm": 0.31358352303504944, + "learning_rate": 0.00021883395940372884, + "loss": 0.6399201202392578, + "mean_token_accuracy": 0.7952846321463585, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6739526629447937, + "epoch": 0.8594229588704727, + "grad_norm": 0.4029310941696167, + "learning_rate": 0.00025542826699632564, + "loss": 0.6259587860107422, + "mean_token_accuracy": 0.8014272648096085, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6681017802655697, + "epoch": 0.9821976672805403, + "grad_norm": 0.3852674961090088, + "learning_rate": 0.00029202257458892247, + "loss": 0.6253739166259765, + "mean_token_accuracy": 0.8020894029736518, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6994867869785854, + "eval_mean_token_accuracy": 0.7849237663405282, + "eval_not_syn_loss": 0.6411319971084595, + "eval_not_syn_runtime": 110.7069, + "eval_not_syn_samples_per_second": 12.61, + "eval_not_syn_steps_per_second": 1.581, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6563669620241438, + "eval_mean_token_accuracy": 0.8207755245481219, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.6019997000694275, + "eval_syn_runtime": 117.5102, + "eval_syn_samples_per_second": 11.88, + "eval_syn_steps_per_second": 1.489, + "step": 408 + }, + { + "entropy": 0.6358311531809986, + "epoch": 1.1031307550644567, + "grad_norm": 0.34651753306388855, + "learning_rate": 0.00029851770373635705, + "loss": 0.5962887191772461, + "mean_token_accuracy": 0.8082216354796123, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6196171633899212, + "epoch": 1.2259054634745243, + "grad_norm": 0.28438520431518555, + "learning_rate": 0.0002981572760785325, + "loss": 0.5779812622070313, + "mean_token_accuracy": 0.8130937224626541, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.624696860164404, + "epoch": 1.3486801718845918, + "grad_norm": 0.3563199043273926, + "learning_rate": 0.000297524500341339, + "loss": 0.5760621643066406, + "mean_token_accuracy": 0.8123102071881294, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6383913996815681, + "epoch": 1.4714548802946594, + "grad_norm": 0.32306408882141113, + "learning_rate": 0.00029662053428333246, + "loss": 0.5874852371215821, + "mean_token_accuracy": 0.8098450502753258, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6132303845882415, + "epoch": 1.5942295887047269, + "grad_norm": 0.3351016044616699, + "learning_rate": 0.0002954470318466225, + "loss": 0.5624140930175782, + "mean_token_accuracy": 0.8152360209822654, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6104247760772705, + "epoch": 1.7170042971147943, + "grad_norm": 0.25214216113090515, + "learning_rate": 0.00029400614013073664, + "loss": 0.5608636093139648, + "mean_token_accuracy": 0.8154580116271972, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6093999670445919, + "epoch": 1.839779005524862, + "grad_norm": 0.25383421778678894, + "learning_rate": 0.0002923004954641779, + "loss": 0.5591114044189454, + "mean_token_accuracy": 0.817192807495594, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6029203486442566, + "epoch": 1.9625537139349294, + "grad_norm": 0.25239020586013794, + "learning_rate": 0.0002903332185808646, + "loss": 0.5529713821411133, + "mean_token_accuracy": 0.8182568901777267, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6090937239783151, + "eval_mean_token_accuracy": 0.7930077845709664, + "eval_not_syn_loss": 0.5995497107505798, + "eval_not_syn_runtime": 110.3965, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5718999467577253, + "eval_mean_token_accuracy": 0.84101799760546, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5510491728782654, + "eval_syn_runtime": 117.3523, + "eval_syn_samples_per_second": 11.896, + "eval_syn_steps_per_second": 1.491, + "step": 816 + }, + { + "entropy": 0.5504116184517817, + "epoch": 2.0834868017188457, + "grad_norm": 0.23869574069976807, + "learning_rate": 0.0002881079089102777, + "loss": 0.5012085723876953, + "mean_token_accuracy": 0.8299948473267144, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5487727333605289, + "epoch": 2.2062615101289134, + "grad_norm": 0.3087007403373718, + "learning_rate": 0.00028562863799176175, + "loss": 0.4988512802124023, + "mean_token_accuracy": 0.8319525212049484, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5354658082127571, + "epoch": 2.329036218538981, + "grad_norm": 0.2878682613372803, + "learning_rate": 0.00028289994202503066, + "loss": 0.4919636917114258, + "mean_token_accuracy": 0.8349191680550575, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5529813665151596, + "epoch": 2.4518109269490487, + "grad_norm": 0.2625614106655121, + "learning_rate": 0.00027992681357050643, + "loss": 0.5027856063842774, + "mean_token_accuracy": 0.8296491304039955, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5497122646868229, + "epoch": 2.574585635359116, + "grad_norm": 0.271993488073349, + "learning_rate": 0.00027671469241467785, + "loss": 0.49415691375732423, + "mean_token_accuracy": 0.8312779009342194, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5419583600759507, + "epoch": 2.6973603437691835, + "grad_norm": 0.23863548040390015, + "learning_rate": 0.00027326945561719136, + "loss": 0.49570159912109374, + "mean_token_accuracy": 0.8327477470040321, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5491574917733669, + "epoch": 2.820135052179251, + "grad_norm": 0.29003486037254333, + "learning_rate": 0.00026959740675788486, + "loss": 0.4965015411376953, + "mean_token_accuracy": 0.8336276519298553, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.5377901926636696, + "epoch": 2.942909760589319, + "grad_norm": 0.27723413705825806, + "learning_rate": 0.0002657052644034388, + "loss": 0.48964527130126956, + "mean_token_accuracy": 0.8356471425294876, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5576562706061772, + "eval_mean_token_accuracy": 0.817173547404153, + "eval_not_syn_loss": 0.5734513401985168, + "eval_not_syn_runtime": 110.4329, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5267024655001504, + "eval_mean_token_accuracy": 0.8285774500029428, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.544999361038208, + "eval_syn_runtime": 117.3289, + "eval_syn_samples_per_second": 11.898, + "eval_syn_steps_per_second": 1.492, + "step": 1224 + }, + { + "entropy": 0.502170312676938, + "epoch": 3.063842848373235, + "grad_norm": 0.44433876872062683, + "learning_rate": 0.0002616001498147458, + "loss": 0.448189811706543, + "mean_token_accuracy": 0.8443391725496592, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.46903550207614897, + "epoch": 3.1866175567833026, + "grad_norm": 0.2440977245569229, + "learning_rate": 0.0002572895739174909, + "loss": 0.4171265029907227, + "mean_token_accuracy": 0.8530975985527038, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4750825077295303, + "epoch": 3.3093922651933703, + "grad_norm": 0.3219708800315857, + "learning_rate": 0.0002527814235597817, + "loss": 0.4262152099609375, + "mean_token_accuracy": 0.8510907486081123, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.479280876070261, + "epoch": 3.4321669736034375, + "grad_norm": 0.35597166419029236, + "learning_rate": 0.0002480839470819708, + "loss": 0.42484298706054685, + "mean_token_accuracy": 0.8504850694537163, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.48920651733875276, + "epoch": 3.554941682013505, + "grad_norm": 0.2936934232711792, + "learning_rate": 0.00024320573922507465, + "loss": 0.43512439727783203, + "mean_token_accuracy": 0.8497199699282646, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.47630224615335465, + "epoch": 3.677716390423573, + "grad_norm": 0.30153411626815796, + "learning_rate": 0.00023815572540539982, + "loss": 0.4236162567138672, + "mean_token_accuracy": 0.8507336723804474, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.4737357534468174, + "epoch": 3.80049109883364, + "grad_norm": 0.3035024106502533, + "learning_rate": 0.00023294314538414883, + "loss": 0.42273353576660155, + "mean_token_accuracy": 0.8513598147034646, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4796703179180622, + "epoch": 3.9232658072437077, + "grad_norm": 0.2834164500236511, + "learning_rate": 0.0002275775363618849, + "loss": 0.4326186752319336, + "mean_token_accuracy": 0.8495047062635421, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5010706964560917, + "eval_mean_token_accuracy": 0.8040556676047189, + "eval_not_syn_loss": 0.5851709842681885, + "eval_not_syn_runtime": 110.3987, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4733030595098223, + "eval_mean_token_accuracy": 0.8468134031976973, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5429526567459106, + "eval_syn_runtime": 117.3735, + "eval_syn_samples_per_second": 11.894, + "eval_syn_steps_per_second": 1.491, + "step": 1632 + }, + { + "entropy": 0.4337418030966357, + "epoch": 4.044198895027624, + "grad_norm": 0.31872114539146423, + "learning_rate": 0.00022206871552878668, + "loss": 0.3853663635253906, + "mean_token_accuracy": 0.8626761530256514, + "num_tokens": 4602893.0, + "step": 1650 + }, + { + "entropy": 0.39638415291905404, + "epoch": 4.166973603437691, + "grad_norm": 0.4238921105861664, + "learning_rate": 0.00021642676210261927, + "loss": 0.3409381866455078, + "mean_token_accuracy": 0.8753284150362015, + "num_tokens": 4735831.0, + "step": 1700 + }, + { + "entropy": 0.38611642628908155, + "epoch": 4.2897483118477595, + "grad_norm": 0.3248041868209839, + "learning_rate": 0.00021066199888728683, + "loss": 0.3360607147216797, + "mean_token_accuracy": 0.8773713061213493, + "num_tokens": 4880737.0, + "step": 1750 + }, + { + "entropy": 0.3938452216982842, + "epoch": 4.412523020257827, + "grad_norm": 0.33257895708084106, + "learning_rate": 0.00020478497338570733, + "loss": 0.34164436340332033, + "mean_token_accuracy": 0.8741997224092484, + "num_tokens": 5022337.0, + "step": 1800 + }, + { + "entropy": 0.39718300312757493, + "epoch": 4.535297728667894, + "grad_norm": 0.3596663177013397, + "learning_rate": 0.00019880643850156687, + "loss": 0.34481029510498046, + "mean_token_accuracy": 0.8744558349251748, + "num_tokens": 5158628.0, + "step": 1850 + }, + { + "entropy": 0.39785161226987836, + "epoch": 4.658072437077962, + "grad_norm": 0.3325240910053253, + "learning_rate": 0.00019273733286526186, + "loss": 0.34437469482421873, + "mean_token_accuracy": 0.8753779655694962, + "num_tokens": 5295686.0, + "step": 1900 + }, + { + "entropy": 0.3919269675016403, + "epoch": 4.780847145488029, + "grad_norm": 0.33487579226493835, + "learning_rate": 0.00018658876082002678, + "loss": 0.34184349060058594, + "mean_token_accuracy": 0.8743453392386437, + "num_tokens": 5440097.0, + "step": 1950 + }, + { + "entropy": 0.3975044973194599, + "epoch": 4.903621853898097, + "grad_norm": 0.27435103058815, + "learning_rate": 0.00018037197210486505, + "loss": 0.3441514205932617, + "mean_token_accuracy": 0.87376918643713, + "num_tokens": 5582367.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4433087486880166, + "eval_mean_token_accuracy": 0.8042705031803676, + "eval_not_syn_loss": 0.6243218183517456, + "eval_not_syn_runtime": 110.4378, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 5690965.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4196248413835253, + "eval_mean_token_accuracy": 0.8377946128164019, + "eval_num_tokens": 5690965.0, + "eval_syn_loss": 0.5790094137191772, + "eval_syn_runtime": 117.3572, + "eval_syn_samples_per_second": 11.895, + "eval_syn_steps_per_second": 1.491, + "step": 2040 + }, + { + "entropy": 0.3863225542954382, + "epoch": 5.024554941682013, + "grad_norm": 0.4834830164909363, + "learning_rate": 0.00017409834127145627, + "loss": 0.330703125, + "mean_token_accuracy": 0.8809086504926537, + "num_tokens": 5718747.0, + "step": 2050 + }, + { + "entropy": 0.3067230442166328, + "epoch": 5.147329650092081, + "grad_norm": 0.41984865069389343, + "learning_rate": 0.0001677793468727003, + "loss": 0.24890127182006835, + "mean_token_accuracy": 0.9052707189321518, + "num_tokens": 5862951.0, + "step": 2100 + }, + { + "entropy": 0.3144265574961901, + "epoch": 5.270104358502149, + "grad_norm": 0.3354702889919281, + "learning_rate": 0.00016142655046097487, + "loss": 0.25450078964233397, + "mean_token_accuracy": 0.9020006003975868, + "num_tokens": 6000342.0, + "step": 2150 + }, + { + "entropy": 0.3033023314923048, + "epoch": 5.392879066912216, + "grad_norm": 0.4292067289352417, + "learning_rate": 0.00015505157543453375, + "loss": 0.24809816360473633, + "mean_token_accuracy": 0.9048506420850754, + "num_tokens": 6145522.0, + "step": 2200 + }, + { + "entropy": 0.3102876263856888, + "epoch": 5.515653775322283, + "grad_norm": 0.47768503427505493, + "learning_rate": 0.00014866608577074797, + "loss": 0.2563666534423828, + "mean_token_accuracy": 0.901572678387165, + "num_tokens": 6282322.0, + "step": 2250 + }, + { + "entropy": 0.30644357711076736, + "epoch": 5.638428483732351, + "grad_norm": 0.4500775933265686, + "learning_rate": 0.00014228176468510215, + "loss": 0.2527992820739746, + "mean_token_accuracy": 0.9036021012067795, + "num_tokens": 6419205.0, + "step": 2300 + }, + { + "entropy": 0.31263820014894006, + "epoch": 5.7612031921424185, + "grad_norm": 0.36889463663101196, + "learning_rate": 0.00013591029325499087, + "loss": 0.2583304977416992, + "mean_token_accuracy": 0.9022675916552544, + "num_tokens": 6560369.0, + "step": 2350 + }, + { + "entropy": 0.31143927775323393, + "epoch": 5.883977900552486, + "grad_norm": 0.4023212790489197, + "learning_rate": 0.00012956332904742786, + "loss": 0.25584844589233396, + "mean_token_accuracy": 0.9025778490304946, + "num_tokens": 6700409.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.38324279410498485, + "eval_mean_token_accuracy": 0.8128126280648368, + "eval_not_syn_loss": 0.6938011646270752, + "eval_not_syn_runtime": 110.423, + "eval_not_syn_samples_per_second": 12.642, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 6829158.0, + "step": 2448 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3641860605989184, + "eval_mean_token_accuracy": 0.8323914204324995, + "eval_num_tokens": 6829158.0, + "eval_syn_loss": 0.6474246382713318, + "eval_syn_runtime": 117.3904, + "eval_syn_samples_per_second": 11.892, + "eval_syn_steps_per_second": 1.491, + "step": 2448 + }, + { + "entropy": 0.3088354336731325, + "epoch": 6.004910988336403, + "grad_norm": 0.2962503731250763, + "learning_rate": 0.0001232524847897706, + "loss": 0.2500633239746094, + "mean_token_accuracy": 0.9047863828349234, + "num_tokens": 6834835.0, + "step": 2450 + }, + { + "entropy": 0.2266372910887003, + "epoch": 6.12768569674647, + "grad_norm": 0.3523392379283905, + "learning_rate": 0.00011698930712248569, + "loss": 0.16580810546875, + "mean_token_accuracy": 0.9356516122817993, + "num_tokens": 6977951.0, + "step": 2500 + }, + { + "entropy": 0.22805758342146873, + "epoch": 6.250460405156538, + "grad_norm": 0.3685902953147888, + "learning_rate": 0.00011078525547283048, + "loss": 0.16887868881225587, + "mean_token_accuracy": 0.9341216534376144, + "num_tokens": 7121409.0, + "step": 2550 + }, + { + "entropy": 0.2327752672880888, + "epoch": 6.373235113566605, + "grad_norm": 0.4073072671890259, + "learning_rate": 0.00010465168108810399, + "loss": 0.17068164825439452, + "mean_token_accuracy": 0.9341617599129677, + "num_tokens": 7258294.0, + "step": 2600 + }, + { + "entropy": 0.23005703277885914, + "epoch": 6.496009821976672, + "grad_norm": 0.39206013083457947, + "learning_rate": 9.859980626682995e-05, + "loss": 0.1686391067504883, + "mean_token_accuracy": 0.9336268669366836, + "num_tokens": 7400869.0, + "step": 2650 + }, + { + "entropy": 0.22620332822203637, + "epoch": 6.6187845303867405, + "grad_norm": 0.3737471103668213, + "learning_rate": 9.26407038258705e-05, + "loss": 0.1691963005065918, + "mean_token_accuracy": 0.9336274519562722, + "num_tokens": 7537988.0, + "step": 2700 + }, + { + "entropy": 0.23150050349533557, + "epoch": 6.741559238796808, + "grad_norm": 0.4270040690898895, + "learning_rate": 8.678527684103929e-05, + "loss": 0.1724449348449707, + "mean_token_accuracy": 0.9331466138362885, + "num_tokens": 7674717.0, + "step": 2750 + }, + { + "entropy": 0.23078016005456448, + "epoch": 6.864333947206875, + "grad_norm": 0.40093865990638733, + "learning_rate": 8.104423869828156e-05, + "loss": 0.17107765197753907, + "mean_token_accuracy": 0.9330676954984665, + "num_tokens": 7809509.0, + "step": 2800 + }, + { + "entropy": 0.22516858130693435, + "epoch": 6.987108655616943, + "grad_norm": 0.40964797139167786, + "learning_rate": 7.542809349192022e-05, + "loss": 0.1664959144592285, + "mean_token_accuracy": 0.9353445410728455, + "num_tokens": 7953622.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.3169115138905389, + "eval_mean_token_accuracy": 0.8068022135325841, + "eval_not_syn_loss": 0.8101338148117065, + "eval_not_syn_runtime": 110.388, + "eval_not_syn_samples_per_second": 12.646, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 7967351.0, + "step": 2856 + }, + { + "epoch": 7.0, + "eval_entropy": 0.3000860639980861, + "eval_mean_token_accuracy": 0.8291359186172486, + "eval_num_tokens": 7967351.0, + "eval_syn_loss": 0.7574861645698547, + "eval_syn_runtime": 117.345, + "eval_syn_samples_per_second": 11.897, + "eval_syn_steps_per_second": 1.491, + "step": 2856 + }, + { + "entropy": 0.18674430096058675, + "epoch": 7.108041743400859, + "grad_norm": 0.3196325898170471, + "learning_rate": 6.994711680583252e-05, + "loss": 0.12185008049011231, + "mean_token_accuracy": 0.953888118569621, + "num_tokens": 8083803.0, + "step": 2900 + }, + { + "entropy": 0.17206964828073978, + "epoch": 7.230816451810927, + "grad_norm": 0.3761785924434662, + "learning_rate": 6.461133691272128e-05, + "loss": 0.1051278018951416, + "mean_token_accuracy": 0.9581465849280357, + "num_tokens": 8226443.0, + "step": 2950 + }, + { + "entropy": 0.16801628757268192, + "epoch": 7.3535911602209945, + "grad_norm": 0.401041716337204, + "learning_rate": 5.943051642587869e-05, + "loss": 0.10529594421386719, + "mean_token_accuracy": 0.9593242672085762, + "num_tokens": 8368450.0, + "step": 3000 + }, + { + "entropy": 0.16075184397399425, + "epoch": 7.476365868631062, + "grad_norm": 0.45159032940864563, + "learning_rate": 5.441413443701439e-05, + "loss": 0.10433159828186035, + "mean_token_accuracy": 0.9596675795316696, + "num_tokens": 8514445.0, + "step": 3050 + }, + { + "entropy": 0.16641639694571495, + "epoch": 7.59914057704113, + "grad_norm": 0.439954936504364, + "learning_rate": 4.957136917282889e-05, + "loss": 0.10695881843566894, + "mean_token_accuracy": 0.9584988197684288, + "num_tokens": 8651914.0, + "step": 3100 + }, + { + "entropy": 0.1666870591044426, + "epoch": 7.721915285451197, + "grad_norm": 0.46230548620224, + "learning_rate": 4.4911081202065014e-05, + "loss": 0.10541806221008301, + "mean_token_accuracy": 0.9582247936725616, + "num_tokens": 8793102.0, + "step": 3150 + }, + { + "entropy": 0.16819031082093716, + "epoch": 7.844689993861264, + "grad_norm": 0.3943890333175659, + "learning_rate": 4.0441797223762166e-05, + "loss": 0.10882070541381836, + "mean_token_accuracy": 0.9577973237633706, + "num_tokens": 8930862.0, + "step": 3200 + }, + { + "entropy": 0.16455162566155196, + "epoch": 7.967464702271332, + "grad_norm": 0.4292966425418854, + "learning_rate": 3.617169446637561e-05, + "loss": 0.10696003913879394, + "mean_token_accuracy": 0.9589823609590531, + "num_tokens": 9068863.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.27482904161725724, + "eval_mean_token_accuracy": 0.8073075127601623, + "eval_not_syn_loss": 0.936057448387146, + "eval_not_syn_runtime": 110.4123, + "eval_not_syn_samples_per_second": 12.644, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 9105544.0, + "step": 3264 + }, + { + "epoch": 8.0, + "eval_entropy": 0.25925376977239334, + "eval_mean_token_accuracy": 0.8265137771197728, + "eval_num_tokens": 9105544.0, + "eval_syn_loss": 0.882119357585907, + "eval_syn_runtime": 117.3341, + "eval_syn_samples_per_second": 11.898, + "eval_syn_steps_per_second": 1.491, + "step": 3264 + }, + { + "entropy": 0.14927380497988105, + "epoch": 8.088397790055248, + "grad_norm": 0.2612760066986084, + "learning_rate": 3.210858572630498e-05, + "loss": 0.08452132225036621, + "mean_token_accuracy": 0.9678469657292826, + "num_tokens": 9202212.0, + "step": 3300 + }, + { + "entropy": 0.13725149285048246, + "epoch": 8.211172498465316, + "grad_norm": 0.23544830083847046, + "learning_rate": 2.8259905073206182e-05, + "loss": 0.07458745956420898, + "mean_token_accuracy": 0.9708940279483795, + "num_tokens": 9340486.0, + "step": 3350 + }, + { + "entropy": 0.1391688558831811, + "epoch": 8.333947206875383, + "grad_norm": 0.23687265813350677, + "learning_rate": 2.4632694248240548e-05, + "loss": 0.07527658462524414, + "mean_token_accuracy": 0.9697121676802635, + "num_tokens": 9476614.0, + "step": 3400 + }, + { + "entropy": 0.1378477082774043, + "epoch": 8.456721915285451, + "grad_norm": 0.4248947501182556, + "learning_rate": 2.123358978014849e-05, + "loss": 0.07582931995391845, + "mean_token_accuracy": 0.9704358732700348, + "num_tokens": 9614420.0, + "step": 3450 + }, + { + "entropy": 0.12935656122863293, + "epoch": 8.579496623695519, + "grad_norm": 0.2261466085910797, + "learning_rate": 1.8068810842720363e-05, + "loss": 0.07171549797058105, + "mean_token_accuracy": 0.9720115476846695, + "num_tokens": 9764329.0, + "step": 3500 + }, + { + "entropy": 0.137032612003386, + "epoch": 8.702271332105585, + "grad_norm": 0.28648674488067627, + "learning_rate": 1.5144147875880633e-05, + "loss": 0.07382936477661133, + "mean_token_accuracy": 0.9700555896759033, + "num_tokens": 9907026.0, + "step": 3550 + }, + { + "entropy": 0.13177497897297144, + "epoch": 8.825046040515653, + "grad_norm": 0.2325354367494583, + "learning_rate": 1.2464951991205664e-05, + "loss": 0.07414592742919922, + "mean_token_accuracy": 0.9710717841982841, + "num_tokens": 10048663.0, + "step": 3600 + }, + { + "entropy": 0.13154912155121565, + "epoch": 8.947820748925722, + "grad_norm": 0.316993772983551, + "learning_rate": 1.0036125181258958e-05, + "loss": 0.07341423034667968, + "mean_token_accuracy": 0.9714728254079819, + "num_tokens": 10188058.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.24302378271307265, + "eval_mean_token_accuracy": 0.8097811685289655, + "eval_not_syn_loss": 1.0759197473526, + "eval_not_syn_runtime": 110.4176, + "eval_not_syn_samples_per_second": 12.643, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 10243737.0, + "step": 3672 + }, + { + "epoch": 9.0, + "eval_entropy": 0.22846667877265386, + "eval_mean_token_accuracy": 0.8239966991969517, + "eval_num_tokens": 10243737.0, + "eval_syn_loss": 1.0279099941253662, + "eval_syn_runtime": 117.3139, + "eval_syn_samples_per_second": 11.9, + "eval_syn_steps_per_second": 1.492, + "step": 3672 + }, + { + "entropy": 0.12550163607533812, + "epoch": 9.068753836709638, + "grad_norm": 0.24909353256225586, + "learning_rate": 7.862111350656945e-06, + "loss": 0.06767164707183838, + "mean_token_accuracy": 0.9748614950833587, + "num_tokens": 10325246.0, + "step": 3700 + }, + { + "entropy": 0.12745588354766368, + "epoch": 9.191528545119706, + "grad_norm": 0.24493981897830963, + "learning_rate": 5.94688818527594e-06, + "loss": 0.0647413969039917, + "mean_token_accuracy": 0.9748008155822754, + "num_tokens": 10460259.0, + "step": 3750 + }, + { + "entropy": 0.12415395628660918, + "epoch": 9.314303253529772, + "grad_norm": 0.198878213763237, + "learning_rate": 4.293959874476508e-06, + "loss": 0.06433301448822021, + "mean_token_accuracy": 0.9751069861650467, + "num_tokens": 10596880.0, + "step": 3800 + }, + { + "entropy": 0.11996097978204488, + "epoch": 9.43707796193984, + "grad_norm": 0.18801912665367126, + "learning_rate": 2.9063506996611133e-06, + "loss": 0.06347461223602295, + "mean_token_accuracy": 0.975999159514904, + "num_tokens": 10737950.0, + "step": 3850 + }, + { + "entropy": 0.12956236319616438, + "epoch": 9.559852670349908, + "grad_norm": 0.21081438660621643, + "learning_rate": 1.7865995008954344e-06, + "loss": 0.0658436918258667, + "mean_token_accuracy": 0.9740688771009445, + "num_tokens": 10873020.0, + "step": 3900 + }, + { + "entropy": 0.12012777786701917, + "epoch": 9.682627378759975, + "grad_norm": 0.23037217557430267, + "learning_rate": 9.367550317180217e-07, + "loss": 0.0623624324798584, + "mean_token_accuracy": 0.9757008582353592, + "num_tokens": 11014972.0, + "step": 3950 + }, + { + "entropy": 0.12063956683501602, + "epoch": 9.805402087170043, + "grad_norm": 0.22544042766094208, + "learning_rate": 3.5837221063703757e-07, + "loss": 0.0629417085647583, + "mean_token_accuracy": 0.9757507374882698, + "num_tokens": 11156776.0, + "step": 4000 + }, + { + "entropy": 0.11980353830382227, + "epoch": 9.92817679558011, + "grad_norm": 0.2577395737171173, + "learning_rate": 5.2509276172438735e-08, + "loss": 0.06168097972869873, + "mean_token_accuracy": 0.97625951141119, + "num_tokens": 11301352.0, + "step": 4050 + }, + { + "epoch": 10.0, + "eval_entropy": 0.23175379114491598, + "eval_mean_token_accuracy": 0.8106740389551436, + "eval_not_syn_loss": 1.1398290395736694, + "eval_not_syn_runtime": 110.405, + "eval_not_syn_samples_per_second": 12.644, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 11381930.0, + "step": 4080 + }, + { + "epoch": 10.0, + "eval_entropy": 0.2174165860244206, + "eval_mean_token_accuracy": 0.8230881585393633, + "eval_num_tokens": 11381930.0, + "eval_syn_loss": 1.1016745567321777, + "eval_syn_runtime": 117.3221, + "eval_syn_samples_per_second": 11.899, + "eval_syn_steps_per_second": 1.492, + "step": 4080 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.8489438807785165e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9118eb8ff2c647ac8f1acb1826eb1cd73893fe31 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.06743035903930279, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..29b898c35117d38ff3cddb848f0dd3dcca5c8230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-816/trainer_state.json @@ -0,0 +1,238 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 816, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.84347476541996, + "epoch": 0.12277470841006753, + "grad_norm": 0.5139632821083069, + "learning_rate": 3.586242144074487e-05, + "loss": 1.7810469055175782, + "mean_token_accuracy": 0.6252794374525547, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8905406814813613, + "epoch": 0.24554941682013506, + "grad_norm": 0.6561369299888611, + "learning_rate": 7.245672903334166e-05, + "loss": 0.83647705078125, + "mean_token_accuracy": 0.7502138108015061, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7635939045250416, + "epoch": 0.3683241252302026, + "grad_norm": 0.4626609683036804, + "learning_rate": 0.00010905103662593846, + "loss": 0.7156005096435547, + "mean_token_accuracy": 0.7786111453175545, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7165373960137367, + "epoch": 0.4910988336402701, + "grad_norm": 0.4098263680934906, + "learning_rate": 0.00014564534421853526, + "loss": 0.67410400390625, + "mean_token_accuracy": 0.7880693352222443, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7035001173615456, + "epoch": 0.6138735420503376, + "grad_norm": 0.33693116903305054, + "learning_rate": 0.0001822396518111321, + "loss": 0.6591246032714844, + "mean_token_accuracy": 0.7931166198849678, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6862310113012791, + "epoch": 0.7366482504604052, + "grad_norm": 0.31358352303504944, + "learning_rate": 0.00021883395940372884, + "loss": 0.6399201202392578, + "mean_token_accuracy": 0.7952846321463585, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6739526629447937, + "epoch": 0.8594229588704727, + "grad_norm": 0.4029310941696167, + "learning_rate": 0.00025542826699632564, + "loss": 0.6259587860107422, + "mean_token_accuracy": 0.8014272648096085, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6681017802655697, + "epoch": 0.9821976672805403, + "grad_norm": 0.3852674961090088, + "learning_rate": 0.00029202257458892247, + "loss": 0.6253739166259765, + "mean_token_accuracy": 0.8020894029736518, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6994867869785854, + "eval_mean_token_accuracy": 0.7849237663405282, + "eval_not_syn_loss": 0.6411319971084595, + "eval_not_syn_runtime": 110.7069, + "eval_not_syn_samples_per_second": 12.61, + "eval_not_syn_steps_per_second": 1.581, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6563669620241438, + "eval_mean_token_accuracy": 0.8207755245481219, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.6019997000694275, + "eval_syn_runtime": 117.5102, + "eval_syn_samples_per_second": 11.88, + "eval_syn_steps_per_second": 1.489, + "step": 408 + }, + { + "entropy": 0.6358311531809986, + "epoch": 1.1031307550644567, + "grad_norm": 0.34651753306388855, + "learning_rate": 0.00029851770373635705, + "loss": 0.5962887191772461, + "mean_token_accuracy": 0.8082216354796123, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6196171633899212, + "epoch": 1.2259054634745243, + "grad_norm": 0.28438520431518555, + "learning_rate": 0.0002981572760785325, + "loss": 0.5779812622070313, + "mean_token_accuracy": 0.8130937224626541, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.624696860164404, + "epoch": 1.3486801718845918, + "grad_norm": 0.3563199043273926, + "learning_rate": 0.000297524500341339, + "loss": 0.5760621643066406, + "mean_token_accuracy": 0.8123102071881294, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6383913996815681, + "epoch": 1.4714548802946594, + "grad_norm": 0.32306408882141113, + "learning_rate": 0.00029662053428333246, + "loss": 0.5874852371215821, + "mean_token_accuracy": 0.8098450502753258, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6132303845882415, + "epoch": 1.5942295887047269, + "grad_norm": 0.3351016044616699, + "learning_rate": 0.0002954470318466225, + "loss": 0.5624140930175782, + "mean_token_accuracy": 0.8152360209822654, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6104247760772705, + "epoch": 1.7170042971147943, + "grad_norm": 0.25214216113090515, + "learning_rate": 0.00029400614013073664, + "loss": 0.5608636093139648, + "mean_token_accuracy": 0.8154580116271972, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6093999670445919, + "epoch": 1.839779005524862, + "grad_norm": 0.25383421778678894, + "learning_rate": 0.0002923004954641779, + "loss": 0.5591114044189454, + "mean_token_accuracy": 0.817192807495594, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6029203486442566, + "epoch": 1.9625537139349294, + "grad_norm": 0.25239020586013794, + "learning_rate": 0.0002903332185808646, + "loss": 0.5529713821411133, + "mean_token_accuracy": 0.8182568901777267, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6090937239783151, + "eval_mean_token_accuracy": 0.7930077845709664, + "eval_not_syn_loss": 0.5995497107505798, + "eval_not_syn_runtime": 110.3965, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5718999467577253, + "eval_mean_token_accuracy": 0.84101799760546, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5510491728782654, + "eval_syn_runtime": 117.3523, + "eval_syn_samples_per_second": 11.896, + "eval_syn_steps_per_second": 1.491, + "step": 816 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.701594466763469e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}