diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ca603f788fe5502cb606ff13d56ba2e3ad3e9149 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: transformers +model_name: Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1 +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1 + +This model is a fine-tuned version of [Qwen/Qwen3-14B-Base](https://huggingface.co/Qwen/Qwen3-14B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/1bujlxs7) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7cc86ca4e8bbd75077b79feae9f8851515e23206 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: transformers +model_name: Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2 +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2 + +This model is a fine-tuned version of [Qwen/Qwen3-14B-Base](https://huggingface.co/Qwen/Qwen3-14B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/1awltkuf) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9118eb8ff2c647ac8f1acb1826eb1cd73893fe31 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.06743035903930279, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..775efdf9ab7dd50b63e41e2ee2724f9545666b86 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1224/trainer_state.json @@ -0,0 +1,340 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1224, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.84347476541996, + "epoch": 0.12277470841006753, + "grad_norm": 0.5139632821083069, + "learning_rate": 3.586242144074487e-05, + "loss": 1.7810469055175782, + "mean_token_accuracy": 0.6252794374525547, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8905406814813613, + "epoch": 0.24554941682013506, + "grad_norm": 0.6561369299888611, + "learning_rate": 7.245672903334166e-05, + "loss": 0.83647705078125, + "mean_token_accuracy": 0.7502138108015061, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7635939045250416, + "epoch": 0.3683241252302026, + "grad_norm": 0.4626609683036804, + "learning_rate": 0.00010905103662593846, + "loss": 0.7156005096435547, + "mean_token_accuracy": 0.7786111453175545, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7165373960137367, + "epoch": 0.4910988336402701, + "grad_norm": 0.4098263680934906, + "learning_rate": 0.00014564534421853526, + "loss": 0.67410400390625, + "mean_token_accuracy": 0.7880693352222443, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7035001173615456, + "epoch": 0.6138735420503376, + "grad_norm": 0.33693116903305054, + "learning_rate": 0.0001822396518111321, + "loss": 0.6591246032714844, + "mean_token_accuracy": 0.7931166198849678, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6862310113012791, + "epoch": 0.7366482504604052, + "grad_norm": 0.31358352303504944, + "learning_rate": 0.00021883395940372884, + "loss": 0.6399201202392578, + "mean_token_accuracy": 0.7952846321463585, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6739526629447937, + "epoch": 0.8594229588704727, + "grad_norm": 0.4029310941696167, + "learning_rate": 0.00025542826699632564, + "loss": 0.6259587860107422, + "mean_token_accuracy": 0.8014272648096085, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6681017802655697, + "epoch": 0.9821976672805403, + "grad_norm": 0.3852674961090088, + "learning_rate": 0.00029202257458892247, + "loss": 0.6253739166259765, + "mean_token_accuracy": 0.8020894029736518, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6994867869785854, + "eval_mean_token_accuracy": 0.7849237663405282, + "eval_not_syn_loss": 0.6411319971084595, + "eval_not_syn_runtime": 110.7069, + "eval_not_syn_samples_per_second": 12.61, + "eval_not_syn_steps_per_second": 1.581, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6563669620241438, + "eval_mean_token_accuracy": 0.8207755245481219, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.6019997000694275, + "eval_syn_runtime": 117.5102, + "eval_syn_samples_per_second": 11.88, + "eval_syn_steps_per_second": 1.489, + "step": 408 + }, + { + "entropy": 0.6358311531809986, + "epoch": 1.1031307550644567, + "grad_norm": 0.34651753306388855, + "learning_rate": 0.00029851770373635705, + "loss": 0.5962887191772461, + "mean_token_accuracy": 0.8082216354796123, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6196171633899212, + "epoch": 1.2259054634745243, + "grad_norm": 0.28438520431518555, + "learning_rate": 0.0002981572760785325, + "loss": 0.5779812622070313, + "mean_token_accuracy": 0.8130937224626541, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.624696860164404, + "epoch": 1.3486801718845918, + "grad_norm": 0.3563199043273926, + "learning_rate": 0.000297524500341339, + "loss": 0.5760621643066406, + "mean_token_accuracy": 0.8123102071881294, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6383913996815681, + "epoch": 1.4714548802946594, + "grad_norm": 0.32306408882141113, + "learning_rate": 0.00029662053428333246, + "loss": 0.5874852371215821, + "mean_token_accuracy": 0.8098450502753258, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6132303845882415, + "epoch": 1.5942295887047269, + "grad_norm": 0.3351016044616699, + "learning_rate": 0.0002954470318466225, + "loss": 0.5624140930175782, + "mean_token_accuracy": 0.8152360209822654, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6104247760772705, + "epoch": 1.7170042971147943, + "grad_norm": 0.25214216113090515, + "learning_rate": 0.00029400614013073664, + "loss": 0.5608636093139648, + "mean_token_accuracy": 0.8154580116271972, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6093999670445919, + "epoch": 1.839779005524862, + "grad_norm": 0.25383421778678894, + "learning_rate": 0.0002923004954641779, + "loss": 0.5591114044189454, + "mean_token_accuracy": 0.817192807495594, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6029203486442566, + "epoch": 1.9625537139349294, + "grad_norm": 0.25239020586013794, + "learning_rate": 0.0002903332185808646, + "loss": 0.5529713821411133, + "mean_token_accuracy": 0.8182568901777267, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6090937239783151, + "eval_mean_token_accuracy": 0.7930077845709664, + "eval_not_syn_loss": 0.5995497107505798, + "eval_not_syn_runtime": 110.3965, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5718999467577253, + "eval_mean_token_accuracy": 0.84101799760546, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5510491728782654, + "eval_syn_runtime": 117.3523, + "eval_syn_samples_per_second": 11.896, + "eval_syn_steps_per_second": 1.491, + "step": 816 + }, + { + "entropy": 0.5504116184517817, + "epoch": 2.0834868017188457, + "grad_norm": 0.23869574069976807, + "learning_rate": 0.0002881079089102777, + "loss": 0.5012085723876953, + "mean_token_accuracy": 0.8299948473267144, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5487727333605289, + "epoch": 2.2062615101289134, + "grad_norm": 0.3087007403373718, + "learning_rate": 0.00028562863799176175, + "loss": 0.4988512802124023, + "mean_token_accuracy": 0.8319525212049484, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5354658082127571, + "epoch": 2.329036218538981, + "grad_norm": 0.2878682613372803, + "learning_rate": 0.00028289994202503066, + "loss": 0.4919636917114258, + "mean_token_accuracy": 0.8349191680550575, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5529813665151596, + "epoch": 2.4518109269490487, + "grad_norm": 0.2625614106655121, + "learning_rate": 0.00027992681357050643, + "loss": 0.5027856063842774, + "mean_token_accuracy": 0.8296491304039955, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5497122646868229, + "epoch": 2.574585635359116, + "grad_norm": 0.271993488073349, + "learning_rate": 0.00027671469241467785, + "loss": 0.49415691375732423, + "mean_token_accuracy": 0.8312779009342194, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5419583600759507, + "epoch": 2.6973603437691835, + "grad_norm": 0.23863548040390015, + "learning_rate": 0.00027326945561719136, + "loss": 0.49570159912109374, + "mean_token_accuracy": 0.8327477470040321, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5491574917733669, + "epoch": 2.820135052179251, + "grad_norm": 0.29003486037254333, + "learning_rate": 0.00026959740675788486, + "loss": 0.4965015411376953, + "mean_token_accuracy": 0.8336276519298553, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.5377901926636696, + "epoch": 2.942909760589319, + "grad_norm": 0.27723413705825806, + "learning_rate": 0.0002657052644034388, + "loss": 0.48964527130126956, + "mean_token_accuracy": 0.8356471425294876, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5576562706061772, + "eval_mean_token_accuracy": 0.817173547404153, + "eval_not_syn_loss": 0.5734513401985168, + "eval_not_syn_runtime": 110.4329, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5267024655001504, + "eval_mean_token_accuracy": 0.8285774500029428, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.544999361038208, + "eval_syn_runtime": 117.3289, + "eval_syn_samples_per_second": 11.898, + "eval_syn_steps_per_second": 1.492, + "step": 1224 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.5555392143591424e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9118eb8ff2c647ac8f1acb1826eb1cd73893fe31 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.06743035903930279, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..484ff87cc344d288f5a978dbf4a54302b9e7dd16 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-1632/trainer_state.json @@ -0,0 +1,442 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1632, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.84347476541996, + "epoch": 0.12277470841006753, + "grad_norm": 0.5139632821083069, + "learning_rate": 3.586242144074487e-05, + "loss": 1.7810469055175782, + "mean_token_accuracy": 0.6252794374525547, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8905406814813613, + "epoch": 0.24554941682013506, + "grad_norm": 0.6561369299888611, + "learning_rate": 7.245672903334166e-05, + "loss": 0.83647705078125, + "mean_token_accuracy": 0.7502138108015061, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7635939045250416, + "epoch": 0.3683241252302026, + "grad_norm": 0.4626609683036804, + "learning_rate": 0.00010905103662593846, + "loss": 0.7156005096435547, + "mean_token_accuracy": 0.7786111453175545, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7165373960137367, + "epoch": 0.4910988336402701, + "grad_norm": 0.4098263680934906, + "learning_rate": 0.00014564534421853526, + "loss": 0.67410400390625, + "mean_token_accuracy": 0.7880693352222443, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7035001173615456, + "epoch": 0.6138735420503376, + "grad_norm": 0.33693116903305054, + "learning_rate": 0.0001822396518111321, + "loss": 0.6591246032714844, + "mean_token_accuracy": 0.7931166198849678, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6862310113012791, + "epoch": 0.7366482504604052, + "grad_norm": 0.31358352303504944, + "learning_rate": 0.00021883395940372884, + "loss": 0.6399201202392578, + "mean_token_accuracy": 0.7952846321463585, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6739526629447937, + "epoch": 0.8594229588704727, + "grad_norm": 0.4029310941696167, + "learning_rate": 0.00025542826699632564, + "loss": 0.6259587860107422, + "mean_token_accuracy": 0.8014272648096085, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6681017802655697, + "epoch": 0.9821976672805403, + "grad_norm": 0.3852674961090088, + "learning_rate": 0.00029202257458892247, + "loss": 0.6253739166259765, + "mean_token_accuracy": 0.8020894029736518, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6994867869785854, + "eval_mean_token_accuracy": 0.7849237663405282, + "eval_not_syn_loss": 0.6411319971084595, + "eval_not_syn_runtime": 110.7069, + "eval_not_syn_samples_per_second": 12.61, + "eval_not_syn_steps_per_second": 1.581, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6563669620241438, + "eval_mean_token_accuracy": 0.8207755245481219, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.6019997000694275, + "eval_syn_runtime": 117.5102, + "eval_syn_samples_per_second": 11.88, + "eval_syn_steps_per_second": 1.489, + "step": 408 + }, + { + "entropy": 0.6358311531809986, + "epoch": 1.1031307550644567, + "grad_norm": 0.34651753306388855, + "learning_rate": 0.00029851770373635705, + "loss": 0.5962887191772461, + "mean_token_accuracy": 0.8082216354796123, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6196171633899212, + "epoch": 1.2259054634745243, + "grad_norm": 0.28438520431518555, + "learning_rate": 0.0002981572760785325, + "loss": 0.5779812622070313, + "mean_token_accuracy": 0.8130937224626541, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.624696860164404, + "epoch": 1.3486801718845918, + "grad_norm": 0.3563199043273926, + "learning_rate": 0.000297524500341339, + "loss": 0.5760621643066406, + "mean_token_accuracy": 0.8123102071881294, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6383913996815681, + "epoch": 1.4714548802946594, + "grad_norm": 0.32306408882141113, + "learning_rate": 0.00029662053428333246, + "loss": 0.5874852371215821, + "mean_token_accuracy": 0.8098450502753258, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6132303845882415, + "epoch": 1.5942295887047269, + "grad_norm": 0.3351016044616699, + "learning_rate": 0.0002954470318466225, + "loss": 0.5624140930175782, + "mean_token_accuracy": 0.8152360209822654, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6104247760772705, + "epoch": 1.7170042971147943, + "grad_norm": 0.25214216113090515, + "learning_rate": 0.00029400614013073664, + "loss": 0.5608636093139648, + "mean_token_accuracy": 0.8154580116271972, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6093999670445919, + "epoch": 1.839779005524862, + "grad_norm": 0.25383421778678894, + "learning_rate": 0.0002923004954641779, + "loss": 0.5591114044189454, + "mean_token_accuracy": 0.817192807495594, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6029203486442566, + "epoch": 1.9625537139349294, + "grad_norm": 0.25239020586013794, + "learning_rate": 0.0002903332185808646, + "loss": 0.5529713821411133, + "mean_token_accuracy": 0.8182568901777267, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6090937239783151, + "eval_mean_token_accuracy": 0.7930077845709664, + "eval_not_syn_loss": 0.5995497107505798, + "eval_not_syn_runtime": 110.3965, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5718999467577253, + "eval_mean_token_accuracy": 0.84101799760546, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5510491728782654, + "eval_syn_runtime": 117.3523, + "eval_syn_samples_per_second": 11.896, + "eval_syn_steps_per_second": 1.491, + "step": 816 + }, + { + "entropy": 0.5504116184517817, + "epoch": 2.0834868017188457, + "grad_norm": 0.23869574069976807, + "learning_rate": 0.0002881079089102777, + "loss": 0.5012085723876953, + "mean_token_accuracy": 0.8299948473267144, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5487727333605289, + "epoch": 2.2062615101289134, + "grad_norm": 0.3087007403373718, + "learning_rate": 0.00028562863799176175, + "loss": 0.4988512802124023, + "mean_token_accuracy": 0.8319525212049484, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5354658082127571, + "epoch": 2.329036218538981, + "grad_norm": 0.2878682613372803, + "learning_rate": 0.00028289994202503066, + "loss": 0.4919636917114258, + "mean_token_accuracy": 0.8349191680550575, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5529813665151596, + "epoch": 2.4518109269490487, + "grad_norm": 0.2625614106655121, + "learning_rate": 0.00027992681357050643, + "loss": 0.5027856063842774, + "mean_token_accuracy": 0.8296491304039955, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5497122646868229, + "epoch": 2.574585635359116, + "grad_norm": 0.271993488073349, + "learning_rate": 0.00027671469241467785, + "loss": 0.49415691375732423, + "mean_token_accuracy": 0.8312779009342194, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5419583600759507, + "epoch": 2.6973603437691835, + "grad_norm": 0.23863548040390015, + "learning_rate": 0.00027326945561719136, + "loss": 0.49570159912109374, + "mean_token_accuracy": 0.8327477470040321, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5491574917733669, + "epoch": 2.820135052179251, + "grad_norm": 0.29003486037254333, + "learning_rate": 0.00026959740675788486, + "loss": 0.4965015411376953, + "mean_token_accuracy": 0.8336276519298553, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.5377901926636696, + "epoch": 2.942909760589319, + "grad_norm": 0.27723413705825806, + "learning_rate": 0.0002657052644034388, + "loss": 0.48964527130126956, + "mean_token_accuracy": 0.8356471425294876, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5576562706061772, + "eval_mean_token_accuracy": 0.817173547404153, + "eval_not_syn_loss": 0.5734513401985168, + "eval_not_syn_runtime": 110.4329, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5267024655001504, + "eval_mean_token_accuracy": 0.8285774500029428, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.544999361038208, + "eval_syn_runtime": 117.3289, + "eval_syn_samples_per_second": 11.898, + "eval_syn_steps_per_second": 1.492, + "step": 1224 + }, + { + "entropy": 0.502170312676938, + "epoch": 3.063842848373235, + "grad_norm": 0.44433876872062683, + "learning_rate": 0.0002616001498147458, + "loss": 0.448189811706543, + "mean_token_accuracy": 0.8443391725496592, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.46903550207614897, + "epoch": 3.1866175567833026, + "grad_norm": 0.2440977245569229, + "learning_rate": 0.0002572895739174909, + "loss": 0.4171265029907227, + "mean_token_accuracy": 0.8530975985527038, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4750825077295303, + "epoch": 3.3093922651933703, + "grad_norm": 0.3219708800315857, + "learning_rate": 0.0002527814235597817, + "loss": 0.4262152099609375, + "mean_token_accuracy": 0.8510907486081123, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.479280876070261, + "epoch": 3.4321669736034375, + "grad_norm": 0.35597166419029236, + "learning_rate": 0.0002480839470819708, + "loss": 0.42484298706054685, + "mean_token_accuracy": 0.8504850694537163, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.48920651733875276, + "epoch": 3.554941682013505, + "grad_norm": 0.2936934232711792, + "learning_rate": 0.00024320573922507465, + "loss": 0.43512439727783203, + "mean_token_accuracy": 0.8497199699282646, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.47630224615335465, + "epoch": 3.677716390423573, + "grad_norm": 0.30153411626815796, + "learning_rate": 0.00023815572540539982, + "loss": 0.4236162567138672, + "mean_token_accuracy": 0.8507336723804474, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.4737357534468174, + "epoch": 3.80049109883364, + "grad_norm": 0.3035024106502533, + "learning_rate": 0.00023294314538414883, + "loss": 0.42273353576660155, + "mean_token_accuracy": 0.8513598147034646, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4796703179180622, + "epoch": 3.9232658072437077, + "grad_norm": 0.2834164500236511, + "learning_rate": 0.0002275775363618849, + "loss": 0.4326186752319336, + "mean_token_accuracy": 0.8495047062635421, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5010706964560917, + "eval_mean_token_accuracy": 0.8040556676047189, + "eval_not_syn_loss": 0.5851709842681885, + "eval_not_syn_runtime": 110.3987, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4733030595098223, + "eval_mean_token_accuracy": 0.8468134031976973, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5429526567459106, + "eval_syn_runtime": 117.3735, + "eval_syn_samples_per_second": 11.894, + "eval_syn_steps_per_second": 1.491, + "step": 1632 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.408322545728922e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9118eb8ff2c647ac8f1acb1826eb1cd73893fe31 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.06743035903930279, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..5d42a57e0f08ef23ce4930a0e7f5493dc111686c --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2040/trainer_state.json @@ -0,0 +1,544 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2040, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.84347476541996, + "epoch": 0.12277470841006753, + "grad_norm": 0.5139632821083069, + "learning_rate": 3.586242144074487e-05, + "loss": 1.7810469055175782, + "mean_token_accuracy": 0.6252794374525547, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8905406814813613, + "epoch": 0.24554941682013506, + "grad_norm": 0.6561369299888611, + "learning_rate": 7.245672903334166e-05, + "loss": 0.83647705078125, + "mean_token_accuracy": 0.7502138108015061, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7635939045250416, + "epoch": 0.3683241252302026, + "grad_norm": 0.4626609683036804, + "learning_rate": 0.00010905103662593846, + "loss": 0.7156005096435547, + "mean_token_accuracy": 0.7786111453175545, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7165373960137367, + "epoch": 0.4910988336402701, + "grad_norm": 0.4098263680934906, + "learning_rate": 0.00014564534421853526, + "loss": 0.67410400390625, + "mean_token_accuracy": 0.7880693352222443, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7035001173615456, + "epoch": 0.6138735420503376, + "grad_norm": 0.33693116903305054, + "learning_rate": 0.0001822396518111321, + "loss": 0.6591246032714844, + "mean_token_accuracy": 0.7931166198849678, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6862310113012791, + "epoch": 0.7366482504604052, + "grad_norm": 0.31358352303504944, + "learning_rate": 0.00021883395940372884, + "loss": 0.6399201202392578, + "mean_token_accuracy": 0.7952846321463585, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6739526629447937, + "epoch": 0.8594229588704727, + "grad_norm": 0.4029310941696167, + "learning_rate": 0.00025542826699632564, + "loss": 0.6259587860107422, + "mean_token_accuracy": 0.8014272648096085, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6681017802655697, + "epoch": 0.9821976672805403, + "grad_norm": 0.3852674961090088, + "learning_rate": 0.00029202257458892247, + "loss": 0.6253739166259765, + "mean_token_accuracy": 0.8020894029736518, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6994867869785854, + "eval_mean_token_accuracy": 0.7849237663405282, + "eval_not_syn_loss": 0.6411319971084595, + "eval_not_syn_runtime": 110.7069, + "eval_not_syn_samples_per_second": 12.61, + "eval_not_syn_steps_per_second": 1.581, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6563669620241438, + "eval_mean_token_accuracy": 0.8207755245481219, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.6019997000694275, + "eval_syn_runtime": 117.5102, + "eval_syn_samples_per_second": 11.88, + "eval_syn_steps_per_second": 1.489, + "step": 408 + }, + { + "entropy": 0.6358311531809986, + "epoch": 1.1031307550644567, + "grad_norm": 0.34651753306388855, + "learning_rate": 0.00029851770373635705, + "loss": 0.5962887191772461, + "mean_token_accuracy": 0.8082216354796123, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6196171633899212, + "epoch": 1.2259054634745243, + "grad_norm": 0.28438520431518555, + "learning_rate": 0.0002981572760785325, + "loss": 0.5779812622070313, + "mean_token_accuracy": 0.8130937224626541, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.624696860164404, + "epoch": 1.3486801718845918, + "grad_norm": 0.3563199043273926, + "learning_rate": 0.000297524500341339, + "loss": 0.5760621643066406, + "mean_token_accuracy": 0.8123102071881294, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6383913996815681, + "epoch": 1.4714548802946594, + "grad_norm": 0.32306408882141113, + "learning_rate": 0.00029662053428333246, + "loss": 0.5874852371215821, + "mean_token_accuracy": 0.8098450502753258, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6132303845882415, + "epoch": 1.5942295887047269, + "grad_norm": 0.3351016044616699, + "learning_rate": 0.0002954470318466225, + "loss": 0.5624140930175782, + "mean_token_accuracy": 0.8152360209822654, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6104247760772705, + "epoch": 1.7170042971147943, + "grad_norm": 0.25214216113090515, + "learning_rate": 0.00029400614013073664, + "loss": 0.5608636093139648, + "mean_token_accuracy": 0.8154580116271972, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6093999670445919, + "epoch": 1.839779005524862, + "grad_norm": 0.25383421778678894, + "learning_rate": 0.0002923004954641779, + "loss": 0.5591114044189454, + "mean_token_accuracy": 0.817192807495594, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6029203486442566, + "epoch": 1.9625537139349294, + "grad_norm": 0.25239020586013794, + "learning_rate": 0.0002903332185808646, + "loss": 0.5529713821411133, + "mean_token_accuracy": 0.8182568901777267, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6090937239783151, + "eval_mean_token_accuracy": 0.7930077845709664, + "eval_not_syn_loss": 0.5995497107505798, + "eval_not_syn_runtime": 110.3965, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5718999467577253, + "eval_mean_token_accuracy": 0.84101799760546, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5510491728782654, + "eval_syn_runtime": 117.3523, + "eval_syn_samples_per_second": 11.896, + "eval_syn_steps_per_second": 1.491, + "step": 816 + }, + { + "entropy": 0.5504116184517817, + "epoch": 2.0834868017188457, + "grad_norm": 0.23869574069976807, + "learning_rate": 0.0002881079089102777, + "loss": 0.5012085723876953, + "mean_token_accuracy": 0.8299948473267144, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5487727333605289, + "epoch": 2.2062615101289134, + "grad_norm": 0.3087007403373718, + "learning_rate": 0.00028562863799176175, + "loss": 0.4988512802124023, + "mean_token_accuracy": 0.8319525212049484, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5354658082127571, + "epoch": 2.329036218538981, + "grad_norm": 0.2878682613372803, + "learning_rate": 0.00028289994202503066, + "loss": 0.4919636917114258, + "mean_token_accuracy": 0.8349191680550575, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5529813665151596, + "epoch": 2.4518109269490487, + "grad_norm": 0.2625614106655121, + "learning_rate": 0.00027992681357050643, + "loss": 0.5027856063842774, + "mean_token_accuracy": 0.8296491304039955, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5497122646868229, + "epoch": 2.574585635359116, + "grad_norm": 0.271993488073349, + "learning_rate": 0.00027671469241467785, + "loss": 0.49415691375732423, + "mean_token_accuracy": 0.8312779009342194, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5419583600759507, + "epoch": 2.6973603437691835, + "grad_norm": 0.23863548040390015, + "learning_rate": 0.00027326945561719136, + "loss": 0.49570159912109374, + "mean_token_accuracy": 0.8327477470040321, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5491574917733669, + "epoch": 2.820135052179251, + "grad_norm": 0.29003486037254333, + "learning_rate": 0.00026959740675788486, + "loss": 0.4965015411376953, + "mean_token_accuracy": 0.8336276519298553, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.5377901926636696, + "epoch": 2.942909760589319, + "grad_norm": 0.27723413705825806, + "learning_rate": 0.0002657052644034388, + "loss": 0.48964527130126956, + "mean_token_accuracy": 0.8356471425294876, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5576562706061772, + "eval_mean_token_accuracy": 0.817173547404153, + "eval_not_syn_loss": 0.5734513401985168, + "eval_not_syn_runtime": 110.4329, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5267024655001504, + "eval_mean_token_accuracy": 0.8285774500029428, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.544999361038208, + "eval_syn_runtime": 117.3289, + "eval_syn_samples_per_second": 11.898, + "eval_syn_steps_per_second": 1.492, + "step": 1224 + }, + { + "entropy": 0.502170312676938, + "epoch": 3.063842848373235, + "grad_norm": 0.44433876872062683, + "learning_rate": 0.0002616001498147458, + "loss": 0.448189811706543, + "mean_token_accuracy": 0.8443391725496592, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.46903550207614897, + "epoch": 3.1866175567833026, + "grad_norm": 0.2440977245569229, + "learning_rate": 0.0002572895739174909, + "loss": 0.4171265029907227, + "mean_token_accuracy": 0.8530975985527038, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4750825077295303, + "epoch": 3.3093922651933703, + "grad_norm": 0.3219708800315857, + "learning_rate": 0.0002527814235597817, + "loss": 0.4262152099609375, + "mean_token_accuracy": 0.8510907486081123, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.479280876070261, + "epoch": 3.4321669736034375, + "grad_norm": 0.35597166419029236, + "learning_rate": 0.0002480839470819708, + "loss": 0.42484298706054685, + "mean_token_accuracy": 0.8504850694537163, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.48920651733875276, + "epoch": 3.554941682013505, + "grad_norm": 0.2936934232711792, + "learning_rate": 0.00024320573922507465, + "loss": 0.43512439727783203, + "mean_token_accuracy": 0.8497199699282646, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.47630224615335465, + "epoch": 3.677716390423573, + "grad_norm": 0.30153411626815796, + "learning_rate": 0.00023815572540539982, + "loss": 0.4236162567138672, + "mean_token_accuracy": 0.8507336723804474, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.4737357534468174, + "epoch": 3.80049109883364, + "grad_norm": 0.3035024106502533, + "learning_rate": 0.00023294314538414883, + "loss": 0.42273353576660155, + "mean_token_accuracy": 0.8513598147034646, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4796703179180622, + "epoch": 3.9232658072437077, + "grad_norm": 0.2834164500236511, + "learning_rate": 0.0002275775363618849, + "loss": 0.4326186752319336, + "mean_token_accuracy": 0.8495047062635421, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5010706964560917, + "eval_mean_token_accuracy": 0.8040556676047189, + "eval_not_syn_loss": 0.5851709842681885, + "eval_not_syn_runtime": 110.3987, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4733030595098223, + "eval_mean_token_accuracy": 0.8468134031976973, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5429526567459106, + "eval_syn_runtime": 117.3735, + "eval_syn_samples_per_second": 11.894, + "eval_syn_steps_per_second": 1.491, + "step": 1632 + }, + { + "entropy": 0.4337418030966357, + "epoch": 4.044198895027624, + "grad_norm": 0.31872114539146423, + "learning_rate": 0.00022206871552878668, + "loss": 0.3853663635253906, + "mean_token_accuracy": 0.8626761530256514, + "num_tokens": 4602893.0, + "step": 1650 + }, + { + "entropy": 0.39638415291905404, + "epoch": 4.166973603437691, + "grad_norm": 0.4238921105861664, + "learning_rate": 0.00021642676210261927, + "loss": 0.3409381866455078, + "mean_token_accuracy": 0.8753284150362015, + "num_tokens": 4735831.0, + "step": 1700 + }, + { + "entropy": 0.38611642628908155, + "epoch": 4.2897483118477595, + "grad_norm": 0.3248041868209839, + "learning_rate": 0.00021066199888728683, + "loss": 0.3360607147216797, + "mean_token_accuracy": 0.8773713061213493, + "num_tokens": 4880737.0, + "step": 1750 + }, + { + "entropy": 0.3938452216982842, + "epoch": 4.412523020257827, + "grad_norm": 0.33257895708084106, + "learning_rate": 0.00020478497338570733, + "loss": 0.34164436340332033, + "mean_token_accuracy": 0.8741997224092484, + "num_tokens": 5022337.0, + "step": 1800 + }, + { + "entropy": 0.39718300312757493, + "epoch": 4.535297728667894, + "grad_norm": 0.3596663177013397, + "learning_rate": 0.00019880643850156687, + "loss": 0.34481029510498046, + "mean_token_accuracy": 0.8744558349251748, + "num_tokens": 5158628.0, + "step": 1850 + }, + { + "entropy": 0.39785161226987836, + "epoch": 4.658072437077962, + "grad_norm": 0.3325240910053253, + "learning_rate": 0.00019273733286526186, + "loss": 0.34437469482421873, + "mean_token_accuracy": 0.8753779655694962, + "num_tokens": 5295686.0, + "step": 1900 + }, + { + "entropy": 0.3919269675016403, + "epoch": 4.780847145488029, + "grad_norm": 0.33487579226493835, + "learning_rate": 0.00018658876082002678, + "loss": 0.34184349060058594, + "mean_token_accuracy": 0.8743453392386437, + "num_tokens": 5440097.0, + "step": 1950 + }, + { + "entropy": 0.3975044973194599, + "epoch": 4.903621853898097, + "grad_norm": 0.27435103058815, + "learning_rate": 0.00018037197210486505, + "loss": 0.3441514205932617, + "mean_token_accuracy": 0.87376918643713, + "num_tokens": 5582367.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4433087486880166, + "eval_mean_token_accuracy": 0.8042705031803676, + "eval_not_syn_loss": 0.6243218183517456, + "eval_not_syn_runtime": 110.4378, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 5690965.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4196248413835253, + "eval_mean_token_accuracy": 0.8377946128164019, + "eval_num_tokens": 5690965.0, + "eval_syn_loss": 0.5790094137191772, + "eval_syn_runtime": 117.3572, + "eval_syn_samples_per_second": 11.895, + "eval_syn_steps_per_second": 1.491, + "step": 2040 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.25810110441216e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/chat_template.jinja b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/trainer_state.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0ca397719ae3596633ca41fbd2375c0aa404db82 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2448/trainer_state.json @@ -0,0 +1,646 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2448, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.84347476541996, + "epoch": 0.12277470841006753, + "grad_norm": 0.5139632821083069, + "learning_rate": 3.586242144074487e-05, + "loss": 1.7810469055175782, + "mean_token_accuracy": 0.6252794374525547, + "num_tokens": 141525.0, + "step": 50 + }, + { + "entropy": 0.8905406814813613, + "epoch": 0.24554941682013506, + "grad_norm": 0.6561369299888611, + "learning_rate": 7.245672903334166e-05, + "loss": 0.83647705078125, + "mean_token_accuracy": 0.7502138108015061, + "num_tokens": 276389.0, + "step": 100 + }, + { + "entropy": 0.7635939045250416, + "epoch": 0.3683241252302026, + "grad_norm": 0.4626609683036804, + "learning_rate": 0.00010905103662593846, + "loss": 0.7156005096435547, + "mean_token_accuracy": 0.7786111453175545, + "num_tokens": 422723.0, + "step": 150 + }, + { + "entropy": 0.7165373960137367, + "epoch": 0.4910988336402701, + "grad_norm": 0.4098263680934906, + "learning_rate": 0.00014564534421853526, + "loss": 0.67410400390625, + "mean_token_accuracy": 0.7880693352222443, + "num_tokens": 560353.0, + "step": 200 + }, + { + "entropy": 0.7035001173615456, + "epoch": 0.6138735420503376, + "grad_norm": 0.33693116903305054, + "learning_rate": 0.0001822396518111321, + "loss": 0.6591246032714844, + "mean_token_accuracy": 0.7931166198849678, + "num_tokens": 702231.0, + "step": 250 + }, + { + "entropy": 0.6862310113012791, + "epoch": 0.7366482504604052, + "grad_norm": 0.31358352303504944, + "learning_rate": 0.00021883395940372884, + "loss": 0.6399201202392578, + "mean_token_accuracy": 0.7952846321463585, + "num_tokens": 842451.0, + "step": 300 + }, + { + "entropy": 0.6739526629447937, + "epoch": 0.8594229588704727, + "grad_norm": 0.4029310941696167, + "learning_rate": 0.00025542826699632564, + "loss": 0.6259587860107422, + "mean_token_accuracy": 0.8014272648096085, + "num_tokens": 980824.0, + "step": 350 + }, + { + "entropy": 0.6681017802655697, + "epoch": 0.9821976672805403, + "grad_norm": 0.3852674961090088, + "learning_rate": 0.00029202257458892247, + "loss": 0.6253739166259765, + "mean_token_accuracy": 0.8020894029736518, + "num_tokens": 1118697.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6994867869785854, + "eval_mean_token_accuracy": 0.7849237663405282, + "eval_not_syn_loss": 0.6411319971084595, + "eval_not_syn_runtime": 110.7069, + "eval_not_syn_samples_per_second": 12.61, + "eval_not_syn_steps_per_second": 1.581, + "eval_num_tokens": 1138193.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6563669620241438, + "eval_mean_token_accuracy": 0.8207755245481219, + "eval_num_tokens": 1138193.0, + "eval_syn_loss": 0.6019997000694275, + "eval_syn_runtime": 117.5102, + "eval_syn_samples_per_second": 11.88, + "eval_syn_steps_per_second": 1.489, + "step": 408 + }, + { + "entropy": 0.6358311531809986, + "epoch": 1.1031307550644567, + "grad_norm": 0.34651753306388855, + "learning_rate": 0.00029851770373635705, + "loss": 0.5962887191772461, + "mean_token_accuracy": 0.8082216354796123, + "num_tokens": 1263520.0, + "step": 450 + }, + { + "entropy": 0.6196171633899212, + "epoch": 1.2259054634745243, + "grad_norm": 0.28438520431518555, + "learning_rate": 0.0002981572760785325, + "loss": 0.5779812622070313, + "mean_token_accuracy": 0.8130937224626541, + "num_tokens": 1405904.0, + "step": 500 + }, + { + "entropy": 0.624696860164404, + "epoch": 1.3486801718845918, + "grad_norm": 0.3563199043273926, + "learning_rate": 0.000297524500341339, + "loss": 0.5760621643066406, + "mean_token_accuracy": 0.8123102071881294, + "num_tokens": 1541752.0, + "step": 550 + }, + { + "entropy": 0.6383913996815681, + "epoch": 1.4714548802946594, + "grad_norm": 0.32306408882141113, + "learning_rate": 0.00029662053428333246, + "loss": 0.5874852371215821, + "mean_token_accuracy": 0.8098450502753258, + "num_tokens": 1675695.0, + "step": 600 + }, + { + "entropy": 0.6132303845882415, + "epoch": 1.5942295887047269, + "grad_norm": 0.3351016044616699, + "learning_rate": 0.0002954470318466225, + "loss": 0.5624140930175782, + "mean_token_accuracy": 0.8152360209822654, + "num_tokens": 1814294.0, + "step": 650 + }, + { + "entropy": 0.6104247760772705, + "epoch": 1.7170042971147943, + "grad_norm": 0.25214216113090515, + "learning_rate": 0.00029400614013073664, + "loss": 0.5608636093139648, + "mean_token_accuracy": 0.8154580116271972, + "num_tokens": 1953015.0, + "step": 700 + }, + { + "entropy": 0.6093999670445919, + "epoch": 1.839779005524862, + "grad_norm": 0.25383421778678894, + "learning_rate": 0.0002923004954641779, + "loss": 0.5591114044189454, + "mean_token_accuracy": 0.817192807495594, + "num_tokens": 2095011.0, + "step": 750 + }, + { + "entropy": 0.6029203486442566, + "epoch": 1.9625537139349294, + "grad_norm": 0.25239020586013794, + "learning_rate": 0.0002903332185808646, + "loss": 0.5529713821411133, + "mean_token_accuracy": 0.8182568901777267, + "num_tokens": 2235222.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6090937239783151, + "eval_mean_token_accuracy": 0.7930077845709664, + "eval_not_syn_loss": 0.5995497107505798, + "eval_not_syn_runtime": 110.3965, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 2276386.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5718999467577253, + "eval_mean_token_accuracy": 0.84101799760546, + "eval_num_tokens": 2276386.0, + "eval_syn_loss": 0.5510491728782654, + "eval_syn_runtime": 117.3523, + "eval_syn_samples_per_second": 11.896, + "eval_syn_steps_per_second": 1.491, + "step": 816 + }, + { + "entropy": 0.5504116184517817, + "epoch": 2.0834868017188457, + "grad_norm": 0.23869574069976807, + "learning_rate": 0.0002881079089102777, + "loss": 0.5012085723876953, + "mean_token_accuracy": 0.8299948473267144, + "num_tokens": 2374282.0, + "step": 850 + }, + { + "entropy": 0.5487727333605289, + "epoch": 2.2062615101289134, + "grad_norm": 0.3087007403373718, + "learning_rate": 0.00028562863799176175, + "loss": 0.4988512802124023, + "mean_token_accuracy": 0.8319525212049484, + "num_tokens": 2507968.0, + "step": 900 + }, + { + "entropy": 0.5354658082127571, + "epoch": 2.329036218538981, + "grad_norm": 0.2878682613372803, + "learning_rate": 0.00028289994202503066, + "loss": 0.4919636917114258, + "mean_token_accuracy": 0.8349191680550575, + "num_tokens": 2650824.0, + "step": 950 + }, + { + "entropy": 0.5529813665151596, + "epoch": 2.4518109269490487, + "grad_norm": 0.2625614106655121, + "learning_rate": 0.00027992681357050643, + "loss": 0.5027856063842774, + "mean_token_accuracy": 0.8296491304039955, + "num_tokens": 2784681.0, + "step": 1000 + }, + { + "entropy": 0.5497122646868229, + "epoch": 2.574585635359116, + "grad_norm": 0.271993488073349, + "learning_rate": 0.00027671469241467785, + "loss": 0.49415691375732423, + "mean_token_accuracy": 0.8312779009342194, + "num_tokens": 2923442.0, + "step": 1050 + }, + { + "entropy": 0.5419583600759507, + "epoch": 2.6973603437691835, + "grad_norm": 0.23863548040390015, + "learning_rate": 0.00027326945561719136, + "loss": 0.49570159912109374, + "mean_token_accuracy": 0.8327477470040321, + "num_tokens": 3068468.0, + "step": 1100 + }, + { + "entropy": 0.5491574917733669, + "epoch": 2.820135052179251, + "grad_norm": 0.29003486037254333, + "learning_rate": 0.00026959740675788486, + "loss": 0.4965015411376953, + "mean_token_accuracy": 0.8336276519298553, + "num_tokens": 3206450.0, + "step": 1150 + }, + { + "entropy": 0.5377901926636696, + "epoch": 2.942909760589319, + "grad_norm": 0.27723413705825806, + "learning_rate": 0.0002657052644034388, + "loss": 0.48964527130126956, + "mean_token_accuracy": 0.8356471425294876, + "num_tokens": 3348780.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5576562706061772, + "eval_mean_token_accuracy": 0.817173547404153, + "eval_not_syn_loss": 0.5734513401985168, + "eval_not_syn_runtime": 110.4329, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 3414579.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5267024655001504, + "eval_mean_token_accuracy": 0.8285774500029428, + "eval_num_tokens": 3414579.0, + "eval_syn_loss": 0.544999361038208, + "eval_syn_runtime": 117.3289, + "eval_syn_samples_per_second": 11.898, + "eval_syn_steps_per_second": 1.492, + "step": 1224 + }, + { + "entropy": 0.502170312676938, + "epoch": 3.063842848373235, + "grad_norm": 0.44433876872062683, + "learning_rate": 0.0002616001498147458, + "loss": 0.448189811706543, + "mean_token_accuracy": 0.8443391725496592, + "num_tokens": 3490514.0, + "step": 1250 + }, + { + "entropy": 0.46903550207614897, + "epoch": 3.1866175567833026, + "grad_norm": 0.2440977245569229, + "learning_rate": 0.0002572895739174909, + "loss": 0.4171265029907227, + "mean_token_accuracy": 0.8530975985527038, + "num_tokens": 3626328.0, + "step": 1300 + }, + { + "entropy": 0.4750825077295303, + "epoch": 3.3093922651933703, + "grad_norm": 0.3219708800315857, + "learning_rate": 0.0002527814235597817, + "loss": 0.4262152099609375, + "mean_token_accuracy": 0.8510907486081123, + "num_tokens": 3768118.0, + "step": 1350 + }, + { + "entropy": 0.479280876070261, + "epoch": 3.4321669736034375, + "grad_norm": 0.35597166419029236, + "learning_rate": 0.0002480839470819708, + "loss": 0.42484298706054685, + "mean_token_accuracy": 0.8504850694537163, + "num_tokens": 3904913.0, + "step": 1400 + }, + { + "entropy": 0.48920651733875276, + "epoch": 3.554941682013505, + "grad_norm": 0.2936934232711792, + "learning_rate": 0.00024320573922507465, + "loss": 0.43512439727783203, + "mean_token_accuracy": 0.8497199699282646, + "num_tokens": 4038110.0, + "step": 1450 + }, + { + "entropy": 0.47630224615335465, + "epoch": 3.677716390423573, + "grad_norm": 0.30153411626815796, + "learning_rate": 0.00023815572540539982, + "loss": 0.4236162567138672, + "mean_token_accuracy": 0.8507336723804474, + "num_tokens": 4180486.0, + "step": 1500 + }, + { + "entropy": 0.4737357534468174, + "epoch": 3.80049109883364, + "grad_norm": 0.3035024106502533, + "learning_rate": 0.00023294314538414883, + "loss": 0.42273353576660155, + "mean_token_accuracy": 0.8513598147034646, + "num_tokens": 4322795.0, + "step": 1550 + }, + { + "entropy": 0.4796703179180622, + "epoch": 3.9232658072437077, + "grad_norm": 0.2834164500236511, + "learning_rate": 0.0002275775363618849, + "loss": 0.4326186752319336, + "mean_token_accuracy": 0.8495047062635421, + "num_tokens": 4461389.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5010706964560917, + "eval_mean_token_accuracy": 0.8040556676047189, + "eval_not_syn_loss": 0.5851709842681885, + "eval_not_syn_runtime": 110.3987, + "eval_not_syn_samples_per_second": 12.645, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 4552772.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4733030595098223, + "eval_mean_token_accuracy": 0.8468134031976973, + "eval_num_tokens": 4552772.0, + "eval_syn_loss": 0.5429526567459106, + "eval_syn_runtime": 117.3735, + "eval_syn_samples_per_second": 11.894, + "eval_syn_steps_per_second": 1.491, + "step": 1632 + }, + { + "entropy": 0.4337418030966357, + "epoch": 4.044198895027624, + "grad_norm": 0.31872114539146423, + "learning_rate": 0.00022206871552878668, + "loss": 0.3853663635253906, + "mean_token_accuracy": 0.8626761530256514, + "num_tokens": 4602893.0, + "step": 1650 + }, + { + "entropy": 0.39638415291905404, + "epoch": 4.166973603437691, + "grad_norm": 0.4238921105861664, + "learning_rate": 0.00021642676210261927, + "loss": 0.3409381866455078, + "mean_token_accuracy": 0.8753284150362015, + "num_tokens": 4735831.0, + "step": 1700 + }, + { + "entropy": 0.38611642628908155, + "epoch": 4.2897483118477595, + "grad_norm": 0.3248041868209839, + "learning_rate": 0.00021066199888728683, + "loss": 0.3360607147216797, + "mean_token_accuracy": 0.8773713061213493, + "num_tokens": 4880737.0, + "step": 1750 + }, + { + "entropy": 0.3938452216982842, + "epoch": 4.412523020257827, + "grad_norm": 0.33257895708084106, + "learning_rate": 0.00020478497338570733, + "loss": 0.34164436340332033, + "mean_token_accuracy": 0.8741997224092484, + "num_tokens": 5022337.0, + "step": 1800 + }, + { + "entropy": 0.39718300312757493, + "epoch": 4.535297728667894, + "grad_norm": 0.3596663177013397, + "learning_rate": 0.00019880643850156687, + "loss": 0.34481029510498046, + "mean_token_accuracy": 0.8744558349251748, + "num_tokens": 5158628.0, + "step": 1850 + }, + { + "entropy": 0.39785161226987836, + "epoch": 4.658072437077962, + "grad_norm": 0.3325240910053253, + "learning_rate": 0.00019273733286526186, + "loss": 0.34437469482421873, + "mean_token_accuracy": 0.8753779655694962, + "num_tokens": 5295686.0, + "step": 1900 + }, + { + "entropy": 0.3919269675016403, + "epoch": 4.780847145488029, + "grad_norm": 0.33487579226493835, + "learning_rate": 0.00018658876082002678, + "loss": 0.34184349060058594, + "mean_token_accuracy": 0.8743453392386437, + "num_tokens": 5440097.0, + "step": 1950 + }, + { + "entropy": 0.3975044973194599, + "epoch": 4.903621853898097, + "grad_norm": 0.27435103058815, + "learning_rate": 0.00018037197210486505, + "loss": 0.3441514205932617, + "mean_token_accuracy": 0.87376918643713, + "num_tokens": 5582367.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4433087486880166, + "eval_mean_token_accuracy": 0.8042705031803676, + "eval_not_syn_loss": 0.6243218183517456, + "eval_not_syn_runtime": 110.4378, + "eval_not_syn_samples_per_second": 12.641, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 5690965.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4196248413835253, + "eval_mean_token_accuracy": 0.8377946128164019, + "eval_num_tokens": 5690965.0, + "eval_syn_loss": 0.5790094137191772, + "eval_syn_runtime": 117.3572, + "eval_syn_samples_per_second": 11.895, + "eval_syn_steps_per_second": 1.491, + "step": 2040 + }, + { + "entropy": 0.3863225542954382, + "epoch": 5.024554941682013, + "grad_norm": 0.4834830164909363, + "learning_rate": 0.00017409834127145627, + "loss": 0.330703125, + "mean_token_accuracy": 0.8809086504926537, + "num_tokens": 5718747.0, + "step": 2050 + }, + { + "entropy": 0.3067230442166328, + "epoch": 5.147329650092081, + "grad_norm": 0.41984865069389343, + "learning_rate": 0.0001677793468727003, + "loss": 0.24890127182006835, + "mean_token_accuracy": 0.9052707189321518, + "num_tokens": 5862951.0, + "step": 2100 + }, + { + "entropy": 0.3144265574961901, + "epoch": 5.270104358502149, + "grad_norm": 0.3354702889919281, + "learning_rate": 0.00016142655046097487, + "loss": 0.25450078964233397, + "mean_token_accuracy": 0.9020006003975868, + "num_tokens": 6000342.0, + "step": 2150 + }, + { + "entropy": 0.3033023314923048, + "epoch": 5.392879066912216, + "grad_norm": 0.4292067289352417, + "learning_rate": 0.00015505157543453375, + "loss": 0.24809816360473633, + "mean_token_accuracy": 0.9048506420850754, + "num_tokens": 6145522.0, + "step": 2200 + }, + { + "entropy": 0.3102876263856888, + "epoch": 5.515653775322283, + "grad_norm": 0.47768503427505493, + "learning_rate": 0.00014866608577074797, + "loss": 0.2563666534423828, + "mean_token_accuracy": 0.901572678387165, + "num_tokens": 6282322.0, + "step": 2250 + }, + { + "entropy": 0.30644357711076736, + "epoch": 5.638428483732351, + "grad_norm": 0.4500775933265686, + "learning_rate": 0.00014228176468510215, + "loss": 0.2527992820739746, + "mean_token_accuracy": 0.9036021012067795, + "num_tokens": 6419205.0, + "step": 2300 + }, + { + "entropy": 0.31263820014894006, + "epoch": 5.7612031921424185, + "grad_norm": 0.36889463663101196, + "learning_rate": 0.00013591029325499087, + "loss": 0.2583304977416992, + "mean_token_accuracy": 0.9022675916552544, + "num_tokens": 6560369.0, + "step": 2350 + }, + { + "entropy": 0.31143927775323393, + "epoch": 5.883977900552486, + "grad_norm": 0.4023212790489197, + "learning_rate": 0.00012956332904742786, + "loss": 0.25584844589233396, + "mean_token_accuracy": 0.9025778490304946, + "num_tokens": 6700409.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.38324279410498485, + "eval_mean_token_accuracy": 0.8128126280648368, + "eval_not_syn_loss": 0.6938011646270752, + "eval_not_syn_runtime": 110.423, + "eval_not_syn_samples_per_second": 12.642, + "eval_not_syn_steps_per_second": 1.585, + "eval_num_tokens": 6829158.0, + "step": 2448 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3641860605989184, + "eval_mean_token_accuracy": 0.8323914204324995, + "eval_num_tokens": 6829158.0, + "eval_syn_loss": 0.6474246382713318, + "eval_syn_runtime": 117.3904, + "eval_syn_samples_per_second": 11.892, + "eval_syn_steps_per_second": 1.491, + "step": 2448 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.1107534880889754e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/README.md b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/adapter_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9118eb8ff2c647ac8f1acb1826eb1cd73893fe31 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.06743035903930279, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "gate_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3-14B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test2/checkpoint-2856/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/README.md b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/adapter_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a72a4c543bf8671cb7019360179c8cabb6e5d6c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.021223473447411947, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "gate_proj", + "v_proj", + "up_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/chat_template.jinja b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/tokenizer_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/trainer_state.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..24c57638c048f68705fb15481db3b73d3f758494 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1371/trainer_state.json @@ -0,0 +1,337 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1371, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.002540482878685, + "epoch": 0.10952902519167579, + "grad_norm": 0.5552461743354797, + "learning_rate": 8.109930924389919e-06, + "loss": 1.9067156982421876, + "mean_token_accuracy": 0.6026969534158707, + "num_tokens": 119060.0, + "step": 50 + }, + { + "entropy": 1.3068562260270118, + "epoch": 0.21905805038335158, + "grad_norm": 1.0465189218521118, + "learning_rate": 1.6385370643155144e-05, + "loss": 1.1655167388916015, + "mean_token_accuracy": 0.7124340075254441, + "num_tokens": 242713.0, + "step": 100 + }, + { + "entropy": 0.8696084088087082, + "epoch": 0.32858707557502737, + "grad_norm": 0.6686795949935913, + "learning_rate": 2.4660810361920367e-05, + "loss": 0.8031976318359375, + "mean_token_accuracy": 0.7754033240675926, + "num_tokens": 360160.0, + "step": 150 + }, + { + "entropy": 0.7853927117586136, + "epoch": 0.43811610076670315, + "grad_norm": 0.9354436993598938, + "learning_rate": 3.293625008068559e-05, + "loss": 0.7226885986328125, + "mean_token_accuracy": 0.791197614967823, + "num_tokens": 485403.0, + "step": 200 + }, + { + "entropy": 0.7382483741641045, + "epoch": 0.547645125958379, + "grad_norm": 0.7535398006439209, + "learning_rate": 4.121168979945081e-05, + "loss": 0.6755471801757813, + "mean_token_accuracy": 0.8054334259033203, + "num_tokens": 601578.0, + "step": 250 + }, + { + "entropy": 0.7218390592932701, + "epoch": 0.6571741511500547, + "grad_norm": 0.8160243034362793, + "learning_rate": 4.948712951821604e-05, + "loss": 0.6614183044433594, + "mean_token_accuracy": 0.8060630604624748, + "num_tokens": 713917.0, + "step": 300 + }, + { + "entropy": 0.7012957927584648, + "epoch": 0.7667031763417306, + "grad_norm": 0.7985761165618896, + "learning_rate": 5.776256923698126e-05, + "loss": 0.6415711975097657, + "mean_token_accuracy": 0.8100408402085304, + "num_tokens": 832995.0, + "step": 350 + }, + { + "entropy": 0.6763210469484329, + "epoch": 0.8762322015334063, + "grad_norm": 0.6829700469970703, + "learning_rate": 6.603800895574648e-05, + "loss": 0.6193617248535156, + "mean_token_accuracy": 0.8140933158993721, + "num_tokens": 959304.0, + "step": 400 + }, + { + "entropy": 0.6874829810857773, + "epoch": 0.9857612267250822, + "grad_norm": 0.6983632445335388, + "learning_rate": 7.43134486745117e-05, + "loss": 0.6262834167480469, + "mean_token_accuracy": 0.8163631609082223, + "num_tokens": 1078880.0, + "step": 450 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6085402492492918, + "eval_loss": 0.6388216018676758, + "eval_mean_token_accuracy": 0.8123693073552752, + "eval_num_tokens": 1094474.0, + "eval_runtime": 95.9846, + "eval_samples_per_second": 10.439, + "eval_steps_per_second": 1.313, + "step": 457 + }, + { + "entropy": 0.6471366012337232, + "epoch": 1.0941949616648412, + "grad_norm": 0.7878272533416748, + "learning_rate": 7.561806001041411e-05, + "loss": 0.589820671081543, + "mean_token_accuracy": 0.8218218798589225, + "num_tokens": 1190065.0, + "step": 500 + }, + { + "entropy": 0.6404865515232087, + "epoch": 1.203723986856517, + "grad_norm": 0.7976964712142944, + "learning_rate": 7.554418144822605e-05, + "loss": 0.5836894607543945, + "mean_token_accuracy": 0.8222010856866837, + "num_tokens": 1313079.0, + "step": 550 + }, + { + "entropy": 0.6355892798304558, + "epoch": 1.3132530120481927, + "grad_norm": 0.7428044080734253, + "learning_rate": 7.541528503116934e-05, + "loss": 0.577253189086914, + "mean_token_accuracy": 0.8260676205158234, + "num_tokens": 1428305.0, + "step": 600 + }, + { + "entropy": 0.6170084626972675, + "epoch": 1.4227820372398685, + "grad_norm": 0.6792078614234924, + "learning_rate": 7.523155873870194e-05, + "loss": 0.561871337890625, + "mean_token_accuracy": 0.8307154527306557, + "num_tokens": 1551971.0, + "step": 650 + }, + { + "entropy": 0.6222476975619793, + "epoch": 1.5323110624315444, + "grad_norm": 0.6532447934150696, + "learning_rate": 7.499327051286336e-05, + "loss": 0.5612493515014648, + "mean_token_accuracy": 0.8296262130141259, + "num_tokens": 1673401.0, + "step": 700 + }, + { + "entropy": 0.6251883202791214, + "epoch": 1.6418400876232202, + "grad_norm": 0.7107782959938049, + "learning_rate": 7.47007678675144e-05, + "loss": 0.565279769897461, + "mean_token_accuracy": 0.8299687370657921, + "num_tokens": 1788813.0, + "step": 750 + }, + { + "entropy": 0.6177104935050011, + "epoch": 1.751369112814896, + "grad_norm": 0.7850057482719421, + "learning_rate": 7.435447738153122e-05, + "loss": 0.5564990234375, + "mean_token_accuracy": 0.8291967037320137, + "num_tokens": 1911556.0, + "step": 800 + }, + { + "entropy": 0.6029443763196468, + "epoch": 1.8608981380065717, + "grad_norm": 0.7116957306861877, + "learning_rate": 7.395490407669285e-05, + "loss": 0.5501844406127929, + "mean_token_accuracy": 0.8335196697711944, + "num_tokens": 2029129.0, + "step": 850 + }, + { + "entropy": 0.6042105440795421, + "epoch": 1.9704271631982475, + "grad_norm": 0.6712159514427185, + "learning_rate": 7.350263068116955e-05, + "loss": 0.5517086029052735, + "mean_token_accuracy": 0.8321291375160217, + "num_tokens": 2155392.0, + "step": 900 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5401396741942753, + "eval_loss": 0.5797445774078369, + "eval_mean_token_accuracy": 0.8243241475688087, + "eval_num_tokens": 2188948.0, + "eval_runtime": 95.5407, + "eval_samples_per_second": 10.488, + "eval_steps_per_second": 1.319, + "step": 914 + }, + { + "entropy": 0.5675096463675451, + "epoch": 2.078860898138007, + "grad_norm": 0.5713562369346619, + "learning_rate": 7.299831677968588e-05, + "loss": 0.5120392227172852, + "mean_token_accuracy": 0.8414448993374603, + "num_tokens": 2277503.0, + "step": 950 + }, + { + "entropy": 0.5500032117962838, + "epoch": 2.1883899233296824, + "grad_norm": 0.5951120257377625, + "learning_rate": 7.244269785159817e-05, + "loss": 0.49384498596191406, + "mean_token_accuracy": 0.8470860269665718, + "num_tokens": 2392968.0, + "step": 1000 + }, + { + "entropy": 0.5683873899281024, + "epoch": 2.297918948521358, + "grad_norm": 0.6816521286964417, + "learning_rate": 7.183658419828891e-05, + "loss": 0.5088459014892578, + "mean_token_accuracy": 0.8413213565945625, + "num_tokens": 2507108.0, + "step": 1050 + }, + { + "entropy": 0.5481136417388917, + "epoch": 2.407447973713034, + "grad_norm": 0.6417970657348633, + "learning_rate": 7.118085976144257e-05, + "loss": 0.49456378936767575, + "mean_token_accuracy": 0.8468478980660439, + "num_tokens": 2633824.0, + "step": 1100 + }, + { + "entropy": 0.5413966289162636, + "epoch": 2.5169769989047097, + "grad_norm": 0.631996214389801, + "learning_rate": 7.047648083392619e-05, + "loss": 0.49154373168945314, + "mean_token_accuracy": 0.8461908429861069, + "num_tokens": 2753131.0, + "step": 1150 + }, + { + "entropy": 0.5583206915855408, + "epoch": 2.6265060240963853, + "grad_norm": 0.7409902215003967, + "learning_rate": 6.972447466515462e-05, + "loss": 0.4927285385131836, + "mean_token_accuracy": 0.8451325806975365, + "num_tokens": 2865112.0, + "step": 1200 + }, + { + "entropy": 0.556266717761755, + "epoch": 2.7360350492880614, + "grad_norm": 0.5275787115097046, + "learning_rate": 6.892593796297452e-05, + "loss": 0.499769401550293, + "mean_token_accuracy": 0.8451313543319702, + "num_tokens": 2980972.0, + "step": 1250 + }, + { + "entropy": 0.5362468618154526, + "epoch": 2.845564074479737, + "grad_norm": 0.5450661182403564, + "learning_rate": 6.808203529425189e-05, + "loss": 0.4860528945922852, + "mean_token_accuracy": 0.8479894894361496, + "num_tokens": 3108708.0, + "step": 1300 + }, + { + "entropy": 0.5334719524532556, + "epoch": 2.955093099671413, + "grad_norm": 0.4825150966644287, + "learning_rate": 6.719399738649542e-05, + "loss": 0.48385780334472656, + "mean_token_accuracy": 0.8489498183131218, + "num_tokens": 3232991.0, + "step": 1350 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5111288797287714, + "eval_loss": 0.5738435387611389, + "eval_mean_token_accuracy": 0.8259959235077813, + "eval_num_tokens": 3283422.0, + "eval_runtime": 95.6002, + "eval_samples_per_second": 10.481, + "eval_steps_per_second": 1.318, + "step": 1371 + } + ], + "logging_steps": 50, + "max_steps": 4570, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.4281968293052416e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/README.md b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/adapter_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a72a4c543bf8671cb7019360179c8cabb6e5d6c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.021223473447411947, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "gate_proj", + "v_proj", + "up_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/chat_template.jinja b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/tokenizer_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/trainer_state.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bb06424cc124404ff1a709752ff491558f229553 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-1828/trainer_state.json @@ -0,0 +1,438 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1828, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.002540482878685, + "epoch": 0.10952902519167579, + "grad_norm": 0.5552461743354797, + "learning_rate": 8.109930924389919e-06, + "loss": 1.9067156982421876, + "mean_token_accuracy": 0.6026969534158707, + "num_tokens": 119060.0, + "step": 50 + }, + { + "entropy": 1.3068562260270118, + "epoch": 0.21905805038335158, + "grad_norm": 1.0465189218521118, + "learning_rate": 1.6385370643155144e-05, + "loss": 1.1655167388916015, + "mean_token_accuracy": 0.7124340075254441, + "num_tokens": 242713.0, + "step": 100 + }, + { + "entropy": 0.8696084088087082, + "epoch": 0.32858707557502737, + "grad_norm": 0.6686795949935913, + "learning_rate": 2.4660810361920367e-05, + "loss": 0.8031976318359375, + "mean_token_accuracy": 0.7754033240675926, + "num_tokens": 360160.0, + "step": 150 + }, + { + "entropy": 0.7853927117586136, + "epoch": 0.43811610076670315, + "grad_norm": 0.9354436993598938, + "learning_rate": 3.293625008068559e-05, + "loss": 0.7226885986328125, + "mean_token_accuracy": 0.791197614967823, + "num_tokens": 485403.0, + "step": 200 + }, + { + "entropy": 0.7382483741641045, + "epoch": 0.547645125958379, + "grad_norm": 0.7535398006439209, + "learning_rate": 4.121168979945081e-05, + "loss": 0.6755471801757813, + "mean_token_accuracy": 0.8054334259033203, + "num_tokens": 601578.0, + "step": 250 + }, + { + "entropy": 0.7218390592932701, + "epoch": 0.6571741511500547, + "grad_norm": 0.8160243034362793, + "learning_rate": 4.948712951821604e-05, + "loss": 0.6614183044433594, + "mean_token_accuracy": 0.8060630604624748, + "num_tokens": 713917.0, + "step": 300 + }, + { + "entropy": 0.7012957927584648, + "epoch": 0.7667031763417306, + "grad_norm": 0.7985761165618896, + "learning_rate": 5.776256923698126e-05, + "loss": 0.6415711975097657, + "mean_token_accuracy": 0.8100408402085304, + "num_tokens": 832995.0, + "step": 350 + }, + { + "entropy": 0.6763210469484329, + "epoch": 0.8762322015334063, + "grad_norm": 0.6829700469970703, + "learning_rate": 6.603800895574648e-05, + "loss": 0.6193617248535156, + "mean_token_accuracy": 0.8140933158993721, + "num_tokens": 959304.0, + "step": 400 + }, + { + "entropy": 0.6874829810857773, + "epoch": 0.9857612267250822, + "grad_norm": 0.6983632445335388, + "learning_rate": 7.43134486745117e-05, + "loss": 0.6262834167480469, + "mean_token_accuracy": 0.8163631609082223, + "num_tokens": 1078880.0, + "step": 450 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6085402492492918, + "eval_loss": 0.6388216018676758, + "eval_mean_token_accuracy": 0.8123693073552752, + "eval_num_tokens": 1094474.0, + "eval_runtime": 95.9846, + "eval_samples_per_second": 10.439, + "eval_steps_per_second": 1.313, + "step": 457 + }, + { + "entropy": 0.6471366012337232, + "epoch": 1.0941949616648412, + "grad_norm": 0.7878272533416748, + "learning_rate": 7.561806001041411e-05, + "loss": 0.589820671081543, + "mean_token_accuracy": 0.8218218798589225, + "num_tokens": 1190065.0, + "step": 500 + }, + { + "entropy": 0.6404865515232087, + "epoch": 1.203723986856517, + "grad_norm": 0.7976964712142944, + "learning_rate": 7.554418144822605e-05, + "loss": 0.5836894607543945, + "mean_token_accuracy": 0.8222010856866837, + "num_tokens": 1313079.0, + "step": 550 + }, + { + "entropy": 0.6355892798304558, + "epoch": 1.3132530120481927, + "grad_norm": 0.7428044080734253, + "learning_rate": 7.541528503116934e-05, + "loss": 0.577253189086914, + "mean_token_accuracy": 0.8260676205158234, + "num_tokens": 1428305.0, + "step": 600 + }, + { + "entropy": 0.6170084626972675, + "epoch": 1.4227820372398685, + "grad_norm": 0.6792078614234924, + "learning_rate": 7.523155873870194e-05, + "loss": 0.561871337890625, + "mean_token_accuracy": 0.8307154527306557, + "num_tokens": 1551971.0, + "step": 650 + }, + { + "entropy": 0.6222476975619793, + "epoch": 1.5323110624315444, + "grad_norm": 0.6532447934150696, + "learning_rate": 7.499327051286336e-05, + "loss": 0.5612493515014648, + "mean_token_accuracy": 0.8296262130141259, + "num_tokens": 1673401.0, + "step": 700 + }, + { + "entropy": 0.6251883202791214, + "epoch": 1.6418400876232202, + "grad_norm": 0.7107782959938049, + "learning_rate": 7.47007678675144e-05, + "loss": 0.565279769897461, + "mean_token_accuracy": 0.8299687370657921, + "num_tokens": 1788813.0, + "step": 750 + }, + { + "entropy": 0.6177104935050011, + "epoch": 1.751369112814896, + "grad_norm": 0.7850057482719421, + "learning_rate": 7.435447738153122e-05, + "loss": 0.5564990234375, + "mean_token_accuracy": 0.8291967037320137, + "num_tokens": 1911556.0, + "step": 800 + }, + { + "entropy": 0.6029443763196468, + "epoch": 1.8608981380065717, + "grad_norm": 0.7116957306861877, + "learning_rate": 7.395490407669285e-05, + "loss": 0.5501844406127929, + "mean_token_accuracy": 0.8335196697711944, + "num_tokens": 2029129.0, + "step": 850 + }, + { + "entropy": 0.6042105440795421, + "epoch": 1.9704271631982475, + "grad_norm": 0.6712159514427185, + "learning_rate": 7.350263068116955e-05, + "loss": 0.5517086029052735, + "mean_token_accuracy": 0.8321291375160217, + "num_tokens": 2155392.0, + "step": 900 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5401396741942753, + "eval_loss": 0.5797445774078369, + "eval_mean_token_accuracy": 0.8243241475688087, + "eval_num_tokens": 2188948.0, + "eval_runtime": 95.5407, + "eval_samples_per_second": 10.488, + "eval_steps_per_second": 1.319, + "step": 914 + }, + { + "entropy": 0.5675096463675451, + "epoch": 2.078860898138007, + "grad_norm": 0.5713562369346619, + "learning_rate": 7.299831677968588e-05, + "loss": 0.5120392227172852, + "mean_token_accuracy": 0.8414448993374603, + "num_tokens": 2277503.0, + "step": 950 + }, + { + "entropy": 0.5500032117962838, + "epoch": 2.1883899233296824, + "grad_norm": 0.5951120257377625, + "learning_rate": 7.244269785159817e-05, + "loss": 0.49384498596191406, + "mean_token_accuracy": 0.8470860269665718, + "num_tokens": 2392968.0, + "step": 1000 + }, + { + "entropy": 0.5683873899281024, + "epoch": 2.297918948521358, + "grad_norm": 0.6816521286964417, + "learning_rate": 7.183658419828891e-05, + "loss": 0.5088459014892578, + "mean_token_accuracy": 0.8413213565945625, + "num_tokens": 2507108.0, + "step": 1050 + }, + { + "entropy": 0.5481136417388917, + "epoch": 2.407447973713034, + "grad_norm": 0.6417970657348633, + "learning_rate": 7.118085976144257e-05, + "loss": 0.49456378936767575, + "mean_token_accuracy": 0.8468478980660439, + "num_tokens": 2633824.0, + "step": 1100 + }, + { + "entropy": 0.5413966289162636, + "epoch": 2.5169769989047097, + "grad_norm": 0.631996214389801, + "learning_rate": 7.047648083392619e-05, + "loss": 0.49154373168945314, + "mean_token_accuracy": 0.8461908429861069, + "num_tokens": 2753131.0, + "step": 1150 + }, + { + "entropy": 0.5583206915855408, + "epoch": 2.6265060240963853, + "grad_norm": 0.7409902215003967, + "learning_rate": 6.972447466515462e-05, + "loss": 0.4927285385131836, + "mean_token_accuracy": 0.8451325806975365, + "num_tokens": 2865112.0, + "step": 1200 + }, + { + "entropy": 0.556266717761755, + "epoch": 2.7360350492880614, + "grad_norm": 0.5275787115097046, + "learning_rate": 6.892593796297452e-05, + "loss": 0.499769401550293, + "mean_token_accuracy": 0.8451313543319702, + "num_tokens": 2980972.0, + "step": 1250 + }, + { + "entropy": 0.5362468618154526, + "epoch": 2.845564074479737, + "grad_norm": 0.5450661182403564, + "learning_rate": 6.808203529425189e-05, + "loss": 0.4860528945922852, + "mean_token_accuracy": 0.8479894894361496, + "num_tokens": 3108708.0, + "step": 1300 + }, + { + "entropy": 0.5334719524532556, + "epoch": 2.955093099671413, + "grad_norm": 0.4825150966644287, + "learning_rate": 6.719399738649542e-05, + "loss": 0.48385780334472656, + "mean_token_accuracy": 0.8489498183131218, + "num_tokens": 3232991.0, + "step": 1350 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5111288797287714, + "eval_loss": 0.5738435387611389, + "eval_mean_token_accuracy": 0.8259959235077813, + "eval_num_tokens": 3283422.0, + "eval_runtime": 95.6002, + "eval_samples_per_second": 10.481, + "eval_steps_per_second": 1.318, + "step": 1371 + }, + { + "entropy": 0.5024978819519582, + "epoch": 3.063526834611172, + "grad_norm": 0.6228470802307129, + "learning_rate": 6.626311933299292e-05, + "loss": 0.4451956939697266, + "mean_token_accuracy": 0.8579568441468056, + "num_tokens": 3352672.0, + "step": 1400 + }, + { + "entropy": 0.49351966604590414, + "epoch": 3.1730558598028478, + "grad_norm": 0.648960530757904, + "learning_rate": 6.529075870407823e-05, + "loss": 0.4324279022216797, + "mean_token_accuracy": 0.8607994091510772, + "num_tokens": 3472463.0, + "step": 1450 + }, + { + "entropy": 0.48342301592230796, + "epoch": 3.2825848849945234, + "grad_norm": 0.8443573713302612, + "learning_rate": 6.427833356728302e-05, + "loss": 0.4237791442871094, + "mean_token_accuracy": 0.8643713328242302, + "num_tokens": 3593837.0, + "step": 1500 + }, + { + "entropy": 0.4779162485897541, + "epoch": 3.3921139101861995, + "grad_norm": 0.7882111072540283, + "learning_rate": 6.32273204192609e-05, + "loss": 0.42386363983154296, + "mean_token_accuracy": 0.8635856115818024, + "num_tokens": 3716058.0, + "step": 1550 + }, + { + "entropy": 0.48870255261659623, + "epoch": 3.501642935377875, + "grad_norm": 0.7637454867362976, + "learning_rate": 6.213925203250001e-05, + "loss": 0.4301974105834961, + "mean_token_accuracy": 0.861739870607853, + "num_tokens": 3838629.0, + "step": 1600 + }, + { + "entropy": 0.49970791533589365, + "epoch": 3.6111719605695507, + "grad_norm": 0.7269095182418823, + "learning_rate": 6.101571521996419e-05, + "loss": 0.4372034454345703, + "mean_token_accuracy": 0.8592326313257217, + "num_tokens": 3955462.0, + "step": 1650 + }, + { + "entropy": 0.5049038740992546, + "epoch": 3.7207009857612268, + "grad_norm": 0.6023766398429871, + "learning_rate": 5.98583485209228e-05, + "loss": 0.445910758972168, + "mean_token_accuracy": 0.8579568776488304, + "num_tokens": 4075933.0, + "step": 1700 + }, + { + "entropy": 0.4927579787373543, + "epoch": 3.8302300109529024, + "grad_norm": 0.7742004990577698, + "learning_rate": 5.866883981134422e-05, + "loss": 0.4280668640136719, + "mean_token_accuracy": 0.86241753667593, + "num_tokens": 4192915.0, + "step": 1750 + }, + { + "entropy": 0.5011553263664246, + "epoch": 3.9397590361445785, + "grad_norm": 0.5748111605644226, + "learning_rate": 5.7448923842337736e-05, + "loss": 0.43842597961425783, + "mean_token_accuracy": 0.8605782136321067, + "num_tokens": 4310431.0, + "step": 1800 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4624898036321004, + "eval_loss": 0.590175986289978, + "eval_mean_token_accuracy": 0.8287760076068696, + "eval_num_tokens": 4377896.0, + "eval_runtime": 95.3934, + "eval_samples_per_second": 10.504, + "eval_steps_per_second": 1.321, + "step": 1828 + } + ], + "logging_steps": 50, + "max_steps": 4570, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.235274069462835e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/README.md b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/adapter_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a72a4c543bf8671cb7019360179c8cabb6e5d6c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.021223473447411947, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "gate_proj", + "v_proj", + "up_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/chat_template.jinja b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/tokenizer_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/trainer_state.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2fb62d327f5e9ca7e42126f17e438de183873038 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2285/trainer_state.json @@ -0,0 +1,539 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2285, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.002540482878685, + "epoch": 0.10952902519167579, + "grad_norm": 0.5552461743354797, + "learning_rate": 8.109930924389919e-06, + "loss": 1.9067156982421876, + "mean_token_accuracy": 0.6026969534158707, + "num_tokens": 119060.0, + "step": 50 + }, + { + "entropy": 1.3068562260270118, + "epoch": 0.21905805038335158, + "grad_norm": 1.0465189218521118, + "learning_rate": 1.6385370643155144e-05, + "loss": 1.1655167388916015, + "mean_token_accuracy": 0.7124340075254441, + "num_tokens": 242713.0, + "step": 100 + }, + { + "entropy": 0.8696084088087082, + "epoch": 0.32858707557502737, + "grad_norm": 0.6686795949935913, + "learning_rate": 2.4660810361920367e-05, + "loss": 0.8031976318359375, + "mean_token_accuracy": 0.7754033240675926, + "num_tokens": 360160.0, + "step": 150 + }, + { + "entropy": 0.7853927117586136, + "epoch": 0.43811610076670315, + "grad_norm": 0.9354436993598938, + "learning_rate": 3.293625008068559e-05, + "loss": 0.7226885986328125, + "mean_token_accuracy": 0.791197614967823, + "num_tokens": 485403.0, + "step": 200 + }, + { + "entropy": 0.7382483741641045, + "epoch": 0.547645125958379, + "grad_norm": 0.7535398006439209, + "learning_rate": 4.121168979945081e-05, + "loss": 0.6755471801757813, + "mean_token_accuracy": 0.8054334259033203, + "num_tokens": 601578.0, + "step": 250 + }, + { + "entropy": 0.7218390592932701, + "epoch": 0.6571741511500547, + "grad_norm": 0.8160243034362793, + "learning_rate": 4.948712951821604e-05, + "loss": 0.6614183044433594, + "mean_token_accuracy": 0.8060630604624748, + "num_tokens": 713917.0, + "step": 300 + }, + { + "entropy": 0.7012957927584648, + "epoch": 0.7667031763417306, + "grad_norm": 0.7985761165618896, + "learning_rate": 5.776256923698126e-05, + "loss": 0.6415711975097657, + "mean_token_accuracy": 0.8100408402085304, + "num_tokens": 832995.0, + "step": 350 + }, + { + "entropy": 0.6763210469484329, + "epoch": 0.8762322015334063, + "grad_norm": 0.6829700469970703, + "learning_rate": 6.603800895574648e-05, + "loss": 0.6193617248535156, + "mean_token_accuracy": 0.8140933158993721, + "num_tokens": 959304.0, + "step": 400 + }, + { + "entropy": 0.6874829810857773, + "epoch": 0.9857612267250822, + "grad_norm": 0.6983632445335388, + "learning_rate": 7.43134486745117e-05, + "loss": 0.6262834167480469, + "mean_token_accuracy": 0.8163631609082223, + "num_tokens": 1078880.0, + "step": 450 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6085402492492918, + "eval_loss": 0.6388216018676758, + "eval_mean_token_accuracy": 0.8123693073552752, + "eval_num_tokens": 1094474.0, + "eval_runtime": 95.9846, + "eval_samples_per_second": 10.439, + "eval_steps_per_second": 1.313, + "step": 457 + }, + { + "entropy": 0.6471366012337232, + "epoch": 1.0941949616648412, + "grad_norm": 0.7878272533416748, + "learning_rate": 7.561806001041411e-05, + "loss": 0.589820671081543, + "mean_token_accuracy": 0.8218218798589225, + "num_tokens": 1190065.0, + "step": 500 + }, + { + "entropy": 0.6404865515232087, + "epoch": 1.203723986856517, + "grad_norm": 0.7976964712142944, + "learning_rate": 7.554418144822605e-05, + "loss": 0.5836894607543945, + "mean_token_accuracy": 0.8222010856866837, + "num_tokens": 1313079.0, + "step": 550 + }, + { + "entropy": 0.6355892798304558, + "epoch": 1.3132530120481927, + "grad_norm": 0.7428044080734253, + "learning_rate": 7.541528503116934e-05, + "loss": 0.577253189086914, + "mean_token_accuracy": 0.8260676205158234, + "num_tokens": 1428305.0, + "step": 600 + }, + { + "entropy": 0.6170084626972675, + "epoch": 1.4227820372398685, + "grad_norm": 0.6792078614234924, + "learning_rate": 7.523155873870194e-05, + "loss": 0.561871337890625, + "mean_token_accuracy": 0.8307154527306557, + "num_tokens": 1551971.0, + "step": 650 + }, + { + "entropy": 0.6222476975619793, + "epoch": 1.5323110624315444, + "grad_norm": 0.6532447934150696, + "learning_rate": 7.499327051286336e-05, + "loss": 0.5612493515014648, + "mean_token_accuracy": 0.8296262130141259, + "num_tokens": 1673401.0, + "step": 700 + }, + { + "entropy": 0.6251883202791214, + "epoch": 1.6418400876232202, + "grad_norm": 0.7107782959938049, + "learning_rate": 7.47007678675144e-05, + "loss": 0.565279769897461, + "mean_token_accuracy": 0.8299687370657921, + "num_tokens": 1788813.0, + "step": 750 + }, + { + "entropy": 0.6177104935050011, + "epoch": 1.751369112814896, + "grad_norm": 0.7850057482719421, + "learning_rate": 7.435447738153122e-05, + "loss": 0.5564990234375, + "mean_token_accuracy": 0.8291967037320137, + "num_tokens": 1911556.0, + "step": 800 + }, + { + "entropy": 0.6029443763196468, + "epoch": 1.8608981380065717, + "grad_norm": 0.7116957306861877, + "learning_rate": 7.395490407669285e-05, + "loss": 0.5501844406127929, + "mean_token_accuracy": 0.8335196697711944, + "num_tokens": 2029129.0, + "step": 850 + }, + { + "entropy": 0.6042105440795421, + "epoch": 1.9704271631982475, + "grad_norm": 0.6712159514427185, + "learning_rate": 7.350263068116955e-05, + "loss": 0.5517086029052735, + "mean_token_accuracy": 0.8321291375160217, + "num_tokens": 2155392.0, + "step": 900 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5401396741942753, + "eval_loss": 0.5797445774078369, + "eval_mean_token_accuracy": 0.8243241475688087, + "eval_num_tokens": 2188948.0, + "eval_runtime": 95.5407, + "eval_samples_per_second": 10.488, + "eval_steps_per_second": 1.319, + "step": 914 + }, + { + "entropy": 0.5675096463675451, + "epoch": 2.078860898138007, + "grad_norm": 0.5713562369346619, + "learning_rate": 7.299831677968588e-05, + "loss": 0.5120392227172852, + "mean_token_accuracy": 0.8414448993374603, + "num_tokens": 2277503.0, + "step": 950 + }, + { + "entropy": 0.5500032117962838, + "epoch": 2.1883899233296824, + "grad_norm": 0.5951120257377625, + "learning_rate": 7.244269785159817e-05, + "loss": 0.49384498596191406, + "mean_token_accuracy": 0.8470860269665718, + "num_tokens": 2392968.0, + "step": 1000 + }, + { + "entropy": 0.5683873899281024, + "epoch": 2.297918948521358, + "grad_norm": 0.6816521286964417, + "learning_rate": 7.183658419828891e-05, + "loss": 0.5088459014892578, + "mean_token_accuracy": 0.8413213565945625, + "num_tokens": 2507108.0, + "step": 1050 + }, + { + "entropy": 0.5481136417388917, + "epoch": 2.407447973713034, + "grad_norm": 0.6417970657348633, + "learning_rate": 7.118085976144257e-05, + "loss": 0.49456378936767575, + "mean_token_accuracy": 0.8468478980660439, + "num_tokens": 2633824.0, + "step": 1100 + }, + { + "entropy": 0.5413966289162636, + "epoch": 2.5169769989047097, + "grad_norm": 0.631996214389801, + "learning_rate": 7.047648083392619e-05, + "loss": 0.49154373168945314, + "mean_token_accuracy": 0.8461908429861069, + "num_tokens": 2753131.0, + "step": 1150 + }, + { + "entropy": 0.5583206915855408, + "epoch": 2.6265060240963853, + "grad_norm": 0.7409902215003967, + "learning_rate": 6.972447466515462e-05, + "loss": 0.4927285385131836, + "mean_token_accuracy": 0.8451325806975365, + "num_tokens": 2865112.0, + "step": 1200 + }, + { + "entropy": 0.556266717761755, + "epoch": 2.7360350492880614, + "grad_norm": 0.5275787115097046, + "learning_rate": 6.892593796297452e-05, + "loss": 0.499769401550293, + "mean_token_accuracy": 0.8451313543319702, + "num_tokens": 2980972.0, + "step": 1250 + }, + { + "entropy": 0.5362468618154526, + "epoch": 2.845564074479737, + "grad_norm": 0.5450661182403564, + "learning_rate": 6.808203529425189e-05, + "loss": 0.4860528945922852, + "mean_token_accuracy": 0.8479894894361496, + "num_tokens": 3108708.0, + "step": 1300 + }, + { + "entropy": 0.5334719524532556, + "epoch": 2.955093099671413, + "grad_norm": 0.4825150966644287, + "learning_rate": 6.719399738649542e-05, + "loss": 0.48385780334472656, + "mean_token_accuracy": 0.8489498183131218, + "num_tokens": 3232991.0, + "step": 1350 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5111288797287714, + "eval_loss": 0.5738435387611389, + "eval_mean_token_accuracy": 0.8259959235077813, + "eval_num_tokens": 3283422.0, + "eval_runtime": 95.6002, + "eval_samples_per_second": 10.481, + "eval_steps_per_second": 1.318, + "step": 1371 + }, + { + "entropy": 0.5024978819519582, + "epoch": 3.063526834611172, + "grad_norm": 0.6228470802307129, + "learning_rate": 6.626311933299292e-05, + "loss": 0.4451956939697266, + "mean_token_accuracy": 0.8579568441468056, + "num_tokens": 3352672.0, + "step": 1400 + }, + { + "entropy": 0.49351966604590414, + "epoch": 3.1730558598028478, + "grad_norm": 0.648960530757904, + "learning_rate": 6.529075870407823e-05, + "loss": 0.4324279022216797, + "mean_token_accuracy": 0.8607994091510772, + "num_tokens": 3472463.0, + "step": 1450 + }, + { + "entropy": 0.48342301592230796, + "epoch": 3.2825848849945234, + "grad_norm": 0.8443573713302612, + "learning_rate": 6.427833356728302e-05, + "loss": 0.4237791442871094, + "mean_token_accuracy": 0.8643713328242302, + "num_tokens": 3593837.0, + "step": 1500 + }, + { + "entropy": 0.4779162485897541, + "epoch": 3.3921139101861995, + "grad_norm": 0.7882111072540283, + "learning_rate": 6.32273204192609e-05, + "loss": 0.42386363983154296, + "mean_token_accuracy": 0.8635856115818024, + "num_tokens": 3716058.0, + "step": 1550 + }, + { + "entropy": 0.48870255261659623, + "epoch": 3.501642935377875, + "grad_norm": 0.7637454867362976, + "learning_rate": 6.213925203250001e-05, + "loss": 0.4301974105834961, + "mean_token_accuracy": 0.861739870607853, + "num_tokens": 3838629.0, + "step": 1600 + }, + { + "entropy": 0.49970791533589365, + "epoch": 3.6111719605695507, + "grad_norm": 0.7269095182418823, + "learning_rate": 6.101571521996419e-05, + "loss": 0.4372034454345703, + "mean_token_accuracy": 0.8592326313257217, + "num_tokens": 3955462.0, + "step": 1650 + }, + { + "entropy": 0.5049038740992546, + "epoch": 3.7207009857612268, + "grad_norm": 0.6023766398429871, + "learning_rate": 5.98583485209228e-05, + "loss": 0.445910758972168, + "mean_token_accuracy": 0.8579568776488304, + "num_tokens": 4075933.0, + "step": 1700 + }, + { + "entropy": 0.4927579787373543, + "epoch": 3.8302300109529024, + "grad_norm": 0.7742004990577698, + "learning_rate": 5.866883981134422e-05, + "loss": 0.4280668640136719, + "mean_token_accuracy": 0.86241753667593, + "num_tokens": 4192915.0, + "step": 1750 + }, + { + "entropy": 0.5011553263664246, + "epoch": 3.9397590361445785, + "grad_norm": 0.5748111605644226, + "learning_rate": 5.7448923842337736e-05, + "loss": 0.43842597961425783, + "mean_token_accuracy": 0.8605782136321067, + "num_tokens": 4310431.0, + "step": 1800 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4624898036321004, + "eval_loss": 0.590175986289978, + "eval_mean_token_accuracy": 0.8287760076068696, + "eval_num_tokens": 4377896.0, + "eval_runtime": 95.3934, + "eval_samples_per_second": 10.504, + "eval_steps_per_second": 1.321, + "step": 1828 + }, + { + "entropy": 0.4502198097079691, + "epoch": 4.048192771084337, + "grad_norm": 0.7170541882514954, + "learning_rate": 5.620037971023403e-05, + "loss": 0.38744712829589845, + "mean_token_accuracy": 0.8735785065877317, + "num_tokens": 4430089.0, + "step": 1850 + }, + { + "entropy": 0.41583337262272835, + "epoch": 4.157721796276014, + "grad_norm": 0.9511433243751526, + "learning_rate": 5.4925028261993515e-05, + "loss": 0.3562023162841797, + "mean_token_accuracy": 0.881559683084488, + "num_tokens": 4554484.0, + "step": 1900 + }, + { + "entropy": 0.4208242034912109, + "epoch": 4.267250821467689, + "grad_norm": 0.8741114139556885, + "learning_rate": 5.3624729439726544e-05, + "loss": 0.3612668991088867, + "mean_token_accuracy": 0.8802913293242455, + "num_tokens": 4675472.0, + "step": 1950 + }, + { + "entropy": 0.4188448017835617, + "epoch": 4.376779846659365, + "grad_norm": 1.1690106391906738, + "learning_rate": 5.23013795681983e-05, + "loss": 0.3627183151245117, + "mean_token_accuracy": 0.8805234292149544, + "num_tokens": 4793479.0, + "step": 2000 + }, + { + "entropy": 0.42332509815692904, + "epoch": 4.48630887185104, + "grad_norm": 0.8150995969772339, + "learning_rate": 5.095690858927403e-05, + "loss": 0.3626524353027344, + "mean_token_accuracy": 0.879372145831585, + "num_tokens": 4911343.0, + "step": 2050 + }, + { + "entropy": 0.42394075110554696, + "epoch": 4.595837897042716, + "grad_norm": 0.8282762169837952, + "learning_rate": 4.959327724733778e-05, + "loss": 0.3573355865478516, + "mean_token_accuracy": 0.8799301481246948, + "num_tokens": 5028364.0, + "step": 2100 + }, + { + "entropy": 0.4259473057091236, + "epoch": 4.705366922234392, + "grad_norm": 0.7610743045806885, + "learning_rate": 4.8212474229789754e-05, + "loss": 0.3665072631835937, + "mean_token_accuracy": 0.879306109547615, + "num_tokens": 5146995.0, + "step": 2150 + }, + { + "entropy": 0.42407046899199485, + "epoch": 4.814895947426068, + "grad_norm": 0.6995375156402588, + "learning_rate": 4.681651326679193e-05, + "loss": 0.3689637756347656, + "mean_token_accuracy": 0.878270491361618, + "num_tokens": 5263483.0, + "step": 2200 + }, + { + "entropy": 0.41719858527183534, + "epoch": 4.924424972617744, + "grad_norm": 0.8921851515769958, + "learning_rate": 4.5407430194492145e-05, + "loss": 0.36366527557373046, + "mean_token_accuracy": 0.8810832899808884, + "num_tokens": 5387955.0, + "step": 2250 + }, + { + "epoch": 5.0, + "eval_entropy": 0.40321539318750776, + "eval_loss": 0.6344199776649475, + "eval_mean_token_accuracy": 0.8290003571245406, + "eval_num_tokens": 5472370.0, + "eval_runtime": 95.4906, + "eval_samples_per_second": 10.493, + "eval_steps_per_second": 1.32, + "step": 2285 + } + ], + "logging_steps": 50, + "max_steps": 4570, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.05719219281623e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/README.md b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/adapter_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a72a4c543bf8671cb7019360179c8cabb6e5d6c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.021223473447411947, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "gate_proj", + "v_proj", + "up_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/chat_template.jinja b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/tokenizer_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/trainer_state.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..69d8fb87baa570e1d31f5a6ce627bed988dc06aa --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-2742/trainer_state.json @@ -0,0 +1,640 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2742, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.002540482878685, + "epoch": 0.10952902519167579, + "grad_norm": 0.5552461743354797, + "learning_rate": 8.109930924389919e-06, + "loss": 1.9067156982421876, + "mean_token_accuracy": 0.6026969534158707, + "num_tokens": 119060.0, + "step": 50 + }, + { + "entropy": 1.3068562260270118, + "epoch": 0.21905805038335158, + "grad_norm": 1.0465189218521118, + "learning_rate": 1.6385370643155144e-05, + "loss": 1.1655167388916015, + "mean_token_accuracy": 0.7124340075254441, + "num_tokens": 242713.0, + "step": 100 + }, + { + "entropy": 0.8696084088087082, + "epoch": 0.32858707557502737, + "grad_norm": 0.6686795949935913, + "learning_rate": 2.4660810361920367e-05, + "loss": 0.8031976318359375, + "mean_token_accuracy": 0.7754033240675926, + "num_tokens": 360160.0, + "step": 150 + }, + { + "entropy": 0.7853927117586136, + "epoch": 0.43811610076670315, + "grad_norm": 0.9354436993598938, + "learning_rate": 3.293625008068559e-05, + "loss": 0.7226885986328125, + "mean_token_accuracy": 0.791197614967823, + "num_tokens": 485403.0, + "step": 200 + }, + { + "entropy": 0.7382483741641045, + "epoch": 0.547645125958379, + "grad_norm": 0.7535398006439209, + "learning_rate": 4.121168979945081e-05, + "loss": 0.6755471801757813, + "mean_token_accuracy": 0.8054334259033203, + "num_tokens": 601578.0, + "step": 250 + }, + { + "entropy": 0.7218390592932701, + "epoch": 0.6571741511500547, + "grad_norm": 0.8160243034362793, + "learning_rate": 4.948712951821604e-05, + "loss": 0.6614183044433594, + "mean_token_accuracy": 0.8060630604624748, + "num_tokens": 713917.0, + "step": 300 + }, + { + "entropy": 0.7012957927584648, + "epoch": 0.7667031763417306, + "grad_norm": 0.7985761165618896, + "learning_rate": 5.776256923698126e-05, + "loss": 0.6415711975097657, + "mean_token_accuracy": 0.8100408402085304, + "num_tokens": 832995.0, + "step": 350 + }, + { + "entropy": 0.6763210469484329, + "epoch": 0.8762322015334063, + "grad_norm": 0.6829700469970703, + "learning_rate": 6.603800895574648e-05, + "loss": 0.6193617248535156, + "mean_token_accuracy": 0.8140933158993721, + "num_tokens": 959304.0, + "step": 400 + }, + { + "entropy": 0.6874829810857773, + "epoch": 0.9857612267250822, + "grad_norm": 0.6983632445335388, + "learning_rate": 7.43134486745117e-05, + "loss": 0.6262834167480469, + "mean_token_accuracy": 0.8163631609082223, + "num_tokens": 1078880.0, + "step": 450 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6085402492492918, + "eval_loss": 0.6388216018676758, + "eval_mean_token_accuracy": 0.8123693073552752, + "eval_num_tokens": 1094474.0, + "eval_runtime": 95.9846, + "eval_samples_per_second": 10.439, + "eval_steps_per_second": 1.313, + "step": 457 + }, + { + "entropy": 0.6471366012337232, + "epoch": 1.0941949616648412, + "grad_norm": 0.7878272533416748, + "learning_rate": 7.561806001041411e-05, + "loss": 0.589820671081543, + "mean_token_accuracy": 0.8218218798589225, + "num_tokens": 1190065.0, + "step": 500 + }, + { + "entropy": 0.6404865515232087, + "epoch": 1.203723986856517, + "grad_norm": 0.7976964712142944, + "learning_rate": 7.554418144822605e-05, + "loss": 0.5836894607543945, + "mean_token_accuracy": 0.8222010856866837, + "num_tokens": 1313079.0, + "step": 550 + }, + { + "entropy": 0.6355892798304558, + "epoch": 1.3132530120481927, + "grad_norm": 0.7428044080734253, + "learning_rate": 7.541528503116934e-05, + "loss": 0.577253189086914, + "mean_token_accuracy": 0.8260676205158234, + "num_tokens": 1428305.0, + "step": 600 + }, + { + "entropy": 0.6170084626972675, + "epoch": 1.4227820372398685, + "grad_norm": 0.6792078614234924, + "learning_rate": 7.523155873870194e-05, + "loss": 0.561871337890625, + "mean_token_accuracy": 0.8307154527306557, + "num_tokens": 1551971.0, + "step": 650 + }, + { + "entropy": 0.6222476975619793, + "epoch": 1.5323110624315444, + "grad_norm": 0.6532447934150696, + "learning_rate": 7.499327051286336e-05, + "loss": 0.5612493515014648, + "mean_token_accuracy": 0.8296262130141259, + "num_tokens": 1673401.0, + "step": 700 + }, + { + "entropy": 0.6251883202791214, + "epoch": 1.6418400876232202, + "grad_norm": 0.7107782959938049, + "learning_rate": 7.47007678675144e-05, + "loss": 0.565279769897461, + "mean_token_accuracy": 0.8299687370657921, + "num_tokens": 1788813.0, + "step": 750 + }, + { + "entropy": 0.6177104935050011, + "epoch": 1.751369112814896, + "grad_norm": 0.7850057482719421, + "learning_rate": 7.435447738153122e-05, + "loss": 0.5564990234375, + "mean_token_accuracy": 0.8291967037320137, + "num_tokens": 1911556.0, + "step": 800 + }, + { + "entropy": 0.6029443763196468, + "epoch": 1.8608981380065717, + "grad_norm": 0.7116957306861877, + "learning_rate": 7.395490407669285e-05, + "loss": 0.5501844406127929, + "mean_token_accuracy": 0.8335196697711944, + "num_tokens": 2029129.0, + "step": 850 + }, + { + "entropy": 0.6042105440795421, + "epoch": 1.9704271631982475, + "grad_norm": 0.6712159514427185, + "learning_rate": 7.350263068116955e-05, + "loss": 0.5517086029052735, + "mean_token_accuracy": 0.8321291375160217, + "num_tokens": 2155392.0, + "step": 900 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5401396741942753, + "eval_loss": 0.5797445774078369, + "eval_mean_token_accuracy": 0.8243241475688087, + "eval_num_tokens": 2188948.0, + "eval_runtime": 95.5407, + "eval_samples_per_second": 10.488, + "eval_steps_per_second": 1.319, + "step": 914 + }, + { + "entropy": 0.5675096463675451, + "epoch": 2.078860898138007, + "grad_norm": 0.5713562369346619, + "learning_rate": 7.299831677968588e-05, + "loss": 0.5120392227172852, + "mean_token_accuracy": 0.8414448993374603, + "num_tokens": 2277503.0, + "step": 950 + }, + { + "entropy": 0.5500032117962838, + "epoch": 2.1883899233296824, + "grad_norm": 0.5951120257377625, + "learning_rate": 7.244269785159817e-05, + "loss": 0.49384498596191406, + "mean_token_accuracy": 0.8470860269665718, + "num_tokens": 2392968.0, + "step": 1000 + }, + { + "entropy": 0.5683873899281024, + "epoch": 2.297918948521358, + "grad_norm": 0.6816521286964417, + "learning_rate": 7.183658419828891e-05, + "loss": 0.5088459014892578, + "mean_token_accuracy": 0.8413213565945625, + "num_tokens": 2507108.0, + "step": 1050 + }, + { + "entropy": 0.5481136417388917, + "epoch": 2.407447973713034, + "grad_norm": 0.6417970657348633, + "learning_rate": 7.118085976144257e-05, + "loss": 0.49456378936767575, + "mean_token_accuracy": 0.8468478980660439, + "num_tokens": 2633824.0, + "step": 1100 + }, + { + "entropy": 0.5413966289162636, + "epoch": 2.5169769989047097, + "grad_norm": 0.631996214389801, + "learning_rate": 7.047648083392619e-05, + "loss": 0.49154373168945314, + "mean_token_accuracy": 0.8461908429861069, + "num_tokens": 2753131.0, + "step": 1150 + }, + { + "entropy": 0.5583206915855408, + "epoch": 2.6265060240963853, + "grad_norm": 0.7409902215003967, + "learning_rate": 6.972447466515462e-05, + "loss": 0.4927285385131836, + "mean_token_accuracy": 0.8451325806975365, + "num_tokens": 2865112.0, + "step": 1200 + }, + { + "entropy": 0.556266717761755, + "epoch": 2.7360350492880614, + "grad_norm": 0.5275787115097046, + "learning_rate": 6.892593796297452e-05, + "loss": 0.499769401550293, + "mean_token_accuracy": 0.8451313543319702, + "num_tokens": 2980972.0, + "step": 1250 + }, + { + "entropy": 0.5362468618154526, + "epoch": 2.845564074479737, + "grad_norm": 0.5450661182403564, + "learning_rate": 6.808203529425189e-05, + "loss": 0.4860528945922852, + "mean_token_accuracy": 0.8479894894361496, + "num_tokens": 3108708.0, + "step": 1300 + }, + { + "entropy": 0.5334719524532556, + "epoch": 2.955093099671413, + "grad_norm": 0.4825150966644287, + "learning_rate": 6.719399738649542e-05, + "loss": 0.48385780334472656, + "mean_token_accuracy": 0.8489498183131218, + "num_tokens": 3232991.0, + "step": 1350 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5111288797287714, + "eval_loss": 0.5738435387611389, + "eval_mean_token_accuracy": 0.8259959235077813, + "eval_num_tokens": 3283422.0, + "eval_runtime": 95.6002, + "eval_samples_per_second": 10.481, + "eval_steps_per_second": 1.318, + "step": 1371 + }, + { + "entropy": 0.5024978819519582, + "epoch": 3.063526834611172, + "grad_norm": 0.6228470802307129, + "learning_rate": 6.626311933299292e-05, + "loss": 0.4451956939697266, + "mean_token_accuracy": 0.8579568441468056, + "num_tokens": 3352672.0, + "step": 1400 + }, + { + "entropy": 0.49351966604590414, + "epoch": 3.1730558598028478, + "grad_norm": 0.648960530757904, + "learning_rate": 6.529075870407823e-05, + "loss": 0.4324279022216797, + "mean_token_accuracy": 0.8607994091510772, + "num_tokens": 3472463.0, + "step": 1450 + }, + { + "entropy": 0.48342301592230796, + "epoch": 3.2825848849945234, + "grad_norm": 0.8443573713302612, + "learning_rate": 6.427833356728302e-05, + "loss": 0.4237791442871094, + "mean_token_accuracy": 0.8643713328242302, + "num_tokens": 3593837.0, + "step": 1500 + }, + { + "entropy": 0.4779162485897541, + "epoch": 3.3921139101861995, + "grad_norm": 0.7882111072540283, + "learning_rate": 6.32273204192609e-05, + "loss": 0.42386363983154296, + "mean_token_accuracy": 0.8635856115818024, + "num_tokens": 3716058.0, + "step": 1550 + }, + { + "entropy": 0.48870255261659623, + "epoch": 3.501642935377875, + "grad_norm": 0.7637454867362976, + "learning_rate": 6.213925203250001e-05, + "loss": 0.4301974105834961, + "mean_token_accuracy": 0.861739870607853, + "num_tokens": 3838629.0, + "step": 1600 + }, + { + "entropy": 0.49970791533589365, + "epoch": 3.6111719605695507, + "grad_norm": 0.7269095182418823, + "learning_rate": 6.101571521996419e-05, + "loss": 0.4372034454345703, + "mean_token_accuracy": 0.8592326313257217, + "num_tokens": 3955462.0, + "step": 1650 + }, + { + "entropy": 0.5049038740992546, + "epoch": 3.7207009857612268, + "grad_norm": 0.6023766398429871, + "learning_rate": 5.98583485209228e-05, + "loss": 0.445910758972168, + "mean_token_accuracy": 0.8579568776488304, + "num_tokens": 4075933.0, + "step": 1700 + }, + { + "entropy": 0.4927579787373543, + "epoch": 3.8302300109529024, + "grad_norm": 0.7742004990577698, + "learning_rate": 5.866883981134422e-05, + "loss": 0.4280668640136719, + "mean_token_accuracy": 0.86241753667593, + "num_tokens": 4192915.0, + "step": 1750 + }, + { + "entropy": 0.5011553263664246, + "epoch": 3.9397590361445785, + "grad_norm": 0.5748111605644226, + "learning_rate": 5.7448923842337736e-05, + "loss": 0.43842597961425783, + "mean_token_accuracy": 0.8605782136321067, + "num_tokens": 4310431.0, + "step": 1800 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4624898036321004, + "eval_loss": 0.590175986289978, + "eval_mean_token_accuracy": 0.8287760076068696, + "eval_num_tokens": 4377896.0, + "eval_runtime": 95.3934, + "eval_samples_per_second": 10.504, + "eval_steps_per_second": 1.321, + "step": 1828 + }, + { + "entropy": 0.4502198097079691, + "epoch": 4.048192771084337, + "grad_norm": 0.7170541882514954, + "learning_rate": 5.620037971023403e-05, + "loss": 0.38744712829589845, + "mean_token_accuracy": 0.8735785065877317, + "num_tokens": 4430089.0, + "step": 1850 + }, + { + "entropy": 0.41583337262272835, + "epoch": 4.157721796276014, + "grad_norm": 0.9511433243751526, + "learning_rate": 5.4925028261993515e-05, + "loss": 0.3562023162841797, + "mean_token_accuracy": 0.881559683084488, + "num_tokens": 4554484.0, + "step": 1900 + }, + { + "entropy": 0.4208242034912109, + "epoch": 4.267250821467689, + "grad_norm": 0.8741114139556885, + "learning_rate": 5.3624729439726544e-05, + "loss": 0.3612668991088867, + "mean_token_accuracy": 0.8802913293242455, + "num_tokens": 4675472.0, + "step": 1950 + }, + { + "entropy": 0.4188448017835617, + "epoch": 4.376779846659365, + "grad_norm": 1.1690106391906738, + "learning_rate": 5.23013795681983e-05, + "loss": 0.3627183151245117, + "mean_token_accuracy": 0.8805234292149544, + "num_tokens": 4793479.0, + "step": 2000 + }, + { + "entropy": 0.42332509815692904, + "epoch": 4.48630887185104, + "grad_norm": 0.8150995969772339, + "learning_rate": 5.095690858927403e-05, + "loss": 0.3626524353027344, + "mean_token_accuracy": 0.879372145831585, + "num_tokens": 4911343.0, + "step": 2050 + }, + { + "entropy": 0.42394075110554696, + "epoch": 4.595837897042716, + "grad_norm": 0.8282762169837952, + "learning_rate": 4.959327724733778e-05, + "loss": 0.3573355865478516, + "mean_token_accuracy": 0.8799301481246948, + "num_tokens": 5028364.0, + "step": 2100 + }, + { + "entropy": 0.4259473057091236, + "epoch": 4.705366922234392, + "grad_norm": 0.7610743045806885, + "learning_rate": 4.8212474229789754e-05, + "loss": 0.3665072631835937, + "mean_token_accuracy": 0.879306109547615, + "num_tokens": 5146995.0, + "step": 2150 + }, + { + "entropy": 0.42407046899199485, + "epoch": 4.814895947426068, + "grad_norm": 0.6995375156402588, + "learning_rate": 4.681651326679193e-05, + "loss": 0.3689637756347656, + "mean_token_accuracy": 0.878270491361618, + "num_tokens": 5263483.0, + "step": 2200 + }, + { + "entropy": 0.41719858527183534, + "epoch": 4.924424972617744, + "grad_norm": 0.8921851515769958, + "learning_rate": 4.5407430194492145e-05, + "loss": 0.36366527557373046, + "mean_token_accuracy": 0.8810832899808884, + "num_tokens": 5387955.0, + "step": 2250 + }, + { + "epoch": 5.0, + "eval_entropy": 0.40321539318750776, + "eval_loss": 0.6344199776649475, + "eval_mean_token_accuracy": 0.8290003571245406, + "eval_num_tokens": 5472370.0, + "eval_runtime": 95.4906, + "eval_samples_per_second": 10.493, + "eval_steps_per_second": 1.32, + "step": 2285 + }, + { + "entropy": 0.39761772337887025, + "epoch": 5.032858707557502, + "grad_norm": 0.8363515734672546, + "learning_rate": 4.3987279986009235e-05, + "loss": 0.3363536834716797, + "mean_token_accuracy": 0.8891537577816935, + "num_tokens": 5508058.0, + "step": 2300 + }, + { + "entropy": 0.3406807939708233, + "epoch": 5.142387732749179, + "grad_norm": 0.9140804409980774, + "learning_rate": 4.2558133754509274e-05, + "loss": 0.2747584533691406, + "mean_token_accuracy": 0.9082898917794228, + "num_tokens": 5625635.0, + "step": 2350 + }, + { + "entropy": 0.34594309888780117, + "epoch": 5.2519167579408546, + "grad_norm": 0.9676663875579834, + "learning_rate": 4.112207573274355e-05, + "loss": 0.2821139907836914, + "mean_token_accuracy": 0.9047485241293907, + "num_tokens": 5741941.0, + "step": 2400 + }, + { + "entropy": 0.3354563079029322, + "epoch": 5.36144578313253, + "grad_norm": 1.1732304096221924, + "learning_rate": 3.968120023345335e-05, + "loss": 0.2757284355163574, + "mean_token_accuracy": 0.905784958600998, + "num_tokens": 5864741.0, + "step": 2450 + }, + { + "entropy": 0.3486690762639046, + "epoch": 5.470974808324206, + "grad_norm": 0.9858622550964355, + "learning_rate": 3.823760859507414e-05, + "loss": 0.28488592147827146, + "mean_token_accuracy": 0.9029894617199897, + "num_tokens": 5984809.0, + "step": 2500 + }, + { + "entropy": 0.35116296328604224, + "epoch": 5.580503833515881, + "grad_norm": 0.8295992016792297, + "learning_rate": 3.679340611719382e-05, + "loss": 0.28818355560302733, + "mean_token_accuracy": 0.9026859793066978, + "num_tokens": 6101316.0, + "step": 2550 + }, + { + "entropy": 0.34762901581823824, + "epoch": 5.690032858707557, + "grad_norm": 1.2317472696304321, + "learning_rate": 3.5350698990234046e-05, + "loss": 0.2834972381591797, + "mean_token_accuracy": 0.9035492998361587, + "num_tokens": 6222300.0, + "step": 2600 + }, + { + "entropy": 0.3509623434394598, + "epoch": 5.7995618838992335, + "grad_norm": 1.1071515083312988, + "learning_rate": 3.391159122383239e-05, + "loss": 0.28417932510375976, + "mean_token_accuracy": 0.9017772257328034, + "num_tokens": 6339706.0, + "step": 2650 + }, + { + "entropy": 0.3327887299656868, + "epoch": 5.909090909090909, + "grad_norm": 0.9410860538482666, + "learning_rate": 3.247818157840487e-05, + "loss": 0.27511814117431643, + "mean_token_accuracy": 0.9069757598638535, + "num_tokens": 6466571.0, + "step": 2700 + }, + { + "epoch": 6.0, + "eval_entropy": 0.37359742701999726, + "eval_loss": 0.6801024079322815, + "eval_mean_token_accuracy": 0.8251904424220796, + "eval_num_tokens": 6566844.0, + "eval_runtime": 95.5582, + "eval_samples_per_second": 10.486, + "eval_steps_per_second": 1.319, + "step": 2742 + } + ], + "logging_steps": 50, + "max_steps": 4570, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.087346418987817e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/README.md b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/adapter_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a72a4c543bf8671cb7019360179c8cabb6e5d6c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.021223473447411947, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "gate_proj", + "v_proj", + "up_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/chat_template.jinja b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/tokenizer_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/trainer_state.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fe362f99859f9d666496a28b6ccc96b44b08450c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3199/trainer_state.json @@ -0,0 +1,741 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 3199, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.002540482878685, + "epoch": 0.10952902519167579, + "grad_norm": 0.5552461743354797, + "learning_rate": 8.109930924389919e-06, + "loss": 1.9067156982421876, + "mean_token_accuracy": 0.6026969534158707, + "num_tokens": 119060.0, + "step": 50 + }, + { + "entropy": 1.3068562260270118, + "epoch": 0.21905805038335158, + "grad_norm": 1.0465189218521118, + "learning_rate": 1.6385370643155144e-05, + "loss": 1.1655167388916015, + "mean_token_accuracy": 0.7124340075254441, + "num_tokens": 242713.0, + "step": 100 + }, + { + "entropy": 0.8696084088087082, + "epoch": 0.32858707557502737, + "grad_norm": 0.6686795949935913, + "learning_rate": 2.4660810361920367e-05, + "loss": 0.8031976318359375, + "mean_token_accuracy": 0.7754033240675926, + "num_tokens": 360160.0, + "step": 150 + }, + { + "entropy": 0.7853927117586136, + "epoch": 0.43811610076670315, + "grad_norm": 0.9354436993598938, + "learning_rate": 3.293625008068559e-05, + "loss": 0.7226885986328125, + "mean_token_accuracy": 0.791197614967823, + "num_tokens": 485403.0, + "step": 200 + }, + { + "entropy": 0.7382483741641045, + "epoch": 0.547645125958379, + "grad_norm": 0.7535398006439209, + "learning_rate": 4.121168979945081e-05, + "loss": 0.6755471801757813, + "mean_token_accuracy": 0.8054334259033203, + "num_tokens": 601578.0, + "step": 250 + }, + { + "entropy": 0.7218390592932701, + "epoch": 0.6571741511500547, + "grad_norm": 0.8160243034362793, + "learning_rate": 4.948712951821604e-05, + "loss": 0.6614183044433594, + "mean_token_accuracy": 0.8060630604624748, + "num_tokens": 713917.0, + "step": 300 + }, + { + "entropy": 0.7012957927584648, + "epoch": 0.7667031763417306, + "grad_norm": 0.7985761165618896, + "learning_rate": 5.776256923698126e-05, + "loss": 0.6415711975097657, + "mean_token_accuracy": 0.8100408402085304, + "num_tokens": 832995.0, + "step": 350 + }, + { + "entropy": 0.6763210469484329, + "epoch": 0.8762322015334063, + "grad_norm": 0.6829700469970703, + "learning_rate": 6.603800895574648e-05, + "loss": 0.6193617248535156, + "mean_token_accuracy": 0.8140933158993721, + "num_tokens": 959304.0, + "step": 400 + }, + { + "entropy": 0.6874829810857773, + "epoch": 0.9857612267250822, + "grad_norm": 0.6983632445335388, + "learning_rate": 7.43134486745117e-05, + "loss": 0.6262834167480469, + "mean_token_accuracy": 0.8163631609082223, + "num_tokens": 1078880.0, + "step": 450 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6085402492492918, + "eval_loss": 0.6388216018676758, + "eval_mean_token_accuracy": 0.8123693073552752, + "eval_num_tokens": 1094474.0, + "eval_runtime": 95.9846, + "eval_samples_per_second": 10.439, + "eval_steps_per_second": 1.313, + "step": 457 + }, + { + "entropy": 0.6471366012337232, + "epoch": 1.0941949616648412, + "grad_norm": 0.7878272533416748, + "learning_rate": 7.561806001041411e-05, + "loss": 0.589820671081543, + "mean_token_accuracy": 0.8218218798589225, + "num_tokens": 1190065.0, + "step": 500 + }, + { + "entropy": 0.6404865515232087, + "epoch": 1.203723986856517, + "grad_norm": 0.7976964712142944, + "learning_rate": 7.554418144822605e-05, + "loss": 0.5836894607543945, + "mean_token_accuracy": 0.8222010856866837, + "num_tokens": 1313079.0, + "step": 550 + }, + { + "entropy": 0.6355892798304558, + "epoch": 1.3132530120481927, + "grad_norm": 0.7428044080734253, + "learning_rate": 7.541528503116934e-05, + "loss": 0.577253189086914, + "mean_token_accuracy": 0.8260676205158234, + "num_tokens": 1428305.0, + "step": 600 + }, + { + "entropy": 0.6170084626972675, + "epoch": 1.4227820372398685, + "grad_norm": 0.6792078614234924, + "learning_rate": 7.523155873870194e-05, + "loss": 0.561871337890625, + "mean_token_accuracy": 0.8307154527306557, + "num_tokens": 1551971.0, + "step": 650 + }, + { + "entropy": 0.6222476975619793, + "epoch": 1.5323110624315444, + "grad_norm": 0.6532447934150696, + "learning_rate": 7.499327051286336e-05, + "loss": 0.5612493515014648, + "mean_token_accuracy": 0.8296262130141259, + "num_tokens": 1673401.0, + "step": 700 + }, + { + "entropy": 0.6251883202791214, + "epoch": 1.6418400876232202, + "grad_norm": 0.7107782959938049, + "learning_rate": 7.47007678675144e-05, + "loss": 0.565279769897461, + "mean_token_accuracy": 0.8299687370657921, + "num_tokens": 1788813.0, + "step": 750 + }, + { + "entropy": 0.6177104935050011, + "epoch": 1.751369112814896, + "grad_norm": 0.7850057482719421, + "learning_rate": 7.435447738153122e-05, + "loss": 0.5564990234375, + "mean_token_accuracy": 0.8291967037320137, + "num_tokens": 1911556.0, + "step": 800 + }, + { + "entropy": 0.6029443763196468, + "epoch": 1.8608981380065717, + "grad_norm": 0.7116957306861877, + "learning_rate": 7.395490407669285e-05, + "loss": 0.5501844406127929, + "mean_token_accuracy": 0.8335196697711944, + "num_tokens": 2029129.0, + "step": 850 + }, + { + "entropy": 0.6042105440795421, + "epoch": 1.9704271631982475, + "grad_norm": 0.6712159514427185, + "learning_rate": 7.350263068116955e-05, + "loss": 0.5517086029052735, + "mean_token_accuracy": 0.8321291375160217, + "num_tokens": 2155392.0, + "step": 900 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5401396741942753, + "eval_loss": 0.5797445774078369, + "eval_mean_token_accuracy": 0.8243241475688087, + "eval_num_tokens": 2188948.0, + "eval_runtime": 95.5407, + "eval_samples_per_second": 10.488, + "eval_steps_per_second": 1.319, + "step": 914 + }, + { + "entropy": 0.5675096463675451, + "epoch": 2.078860898138007, + "grad_norm": 0.5713562369346619, + "learning_rate": 7.299831677968588e-05, + "loss": 0.5120392227172852, + "mean_token_accuracy": 0.8414448993374603, + "num_tokens": 2277503.0, + "step": 950 + }, + { + "entropy": 0.5500032117962838, + "epoch": 2.1883899233296824, + "grad_norm": 0.5951120257377625, + "learning_rate": 7.244269785159817e-05, + "loss": 0.49384498596191406, + "mean_token_accuracy": 0.8470860269665718, + "num_tokens": 2392968.0, + "step": 1000 + }, + { + "entropy": 0.5683873899281024, + "epoch": 2.297918948521358, + "grad_norm": 0.6816521286964417, + "learning_rate": 7.183658419828891e-05, + "loss": 0.5088459014892578, + "mean_token_accuracy": 0.8413213565945625, + "num_tokens": 2507108.0, + "step": 1050 + }, + { + "entropy": 0.5481136417388917, + "epoch": 2.407447973713034, + "grad_norm": 0.6417970657348633, + "learning_rate": 7.118085976144257e-05, + "loss": 0.49456378936767575, + "mean_token_accuracy": 0.8468478980660439, + "num_tokens": 2633824.0, + "step": 1100 + }, + { + "entropy": 0.5413966289162636, + "epoch": 2.5169769989047097, + "grad_norm": 0.631996214389801, + "learning_rate": 7.047648083392619e-05, + "loss": 0.49154373168945314, + "mean_token_accuracy": 0.8461908429861069, + "num_tokens": 2753131.0, + "step": 1150 + }, + { + "entropy": 0.5583206915855408, + "epoch": 2.6265060240963853, + "grad_norm": 0.7409902215003967, + "learning_rate": 6.972447466515462e-05, + "loss": 0.4927285385131836, + "mean_token_accuracy": 0.8451325806975365, + "num_tokens": 2865112.0, + "step": 1200 + }, + { + "entropy": 0.556266717761755, + "epoch": 2.7360350492880614, + "grad_norm": 0.5275787115097046, + "learning_rate": 6.892593796297452e-05, + "loss": 0.499769401550293, + "mean_token_accuracy": 0.8451313543319702, + "num_tokens": 2980972.0, + "step": 1250 + }, + { + "entropy": 0.5362468618154526, + "epoch": 2.845564074479737, + "grad_norm": 0.5450661182403564, + "learning_rate": 6.808203529425189e-05, + "loss": 0.4860528945922852, + "mean_token_accuracy": 0.8479894894361496, + "num_tokens": 3108708.0, + "step": 1300 + }, + { + "entropy": 0.5334719524532556, + "epoch": 2.955093099671413, + "grad_norm": 0.4825150966644287, + "learning_rate": 6.719399738649542e-05, + "loss": 0.48385780334472656, + "mean_token_accuracy": 0.8489498183131218, + "num_tokens": 3232991.0, + "step": 1350 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5111288797287714, + "eval_loss": 0.5738435387611389, + "eval_mean_token_accuracy": 0.8259959235077813, + "eval_num_tokens": 3283422.0, + "eval_runtime": 95.6002, + "eval_samples_per_second": 10.481, + "eval_steps_per_second": 1.318, + "step": 1371 + }, + { + "entropy": 0.5024978819519582, + "epoch": 3.063526834611172, + "grad_norm": 0.6228470802307129, + "learning_rate": 6.626311933299292e-05, + "loss": 0.4451956939697266, + "mean_token_accuracy": 0.8579568441468056, + "num_tokens": 3352672.0, + "step": 1400 + }, + { + "entropy": 0.49351966604590414, + "epoch": 3.1730558598028478, + "grad_norm": 0.648960530757904, + "learning_rate": 6.529075870407823e-05, + "loss": 0.4324279022216797, + "mean_token_accuracy": 0.8607994091510772, + "num_tokens": 3472463.0, + "step": 1450 + }, + { + "entropy": 0.48342301592230796, + "epoch": 3.2825848849945234, + "grad_norm": 0.8443573713302612, + "learning_rate": 6.427833356728302e-05, + "loss": 0.4237791442871094, + "mean_token_accuracy": 0.8643713328242302, + "num_tokens": 3593837.0, + "step": 1500 + }, + { + "entropy": 0.4779162485897541, + "epoch": 3.3921139101861995, + "grad_norm": 0.7882111072540283, + "learning_rate": 6.32273204192609e-05, + "loss": 0.42386363983154296, + "mean_token_accuracy": 0.8635856115818024, + "num_tokens": 3716058.0, + "step": 1550 + }, + { + "entropy": 0.48870255261659623, + "epoch": 3.501642935377875, + "grad_norm": 0.7637454867362976, + "learning_rate": 6.213925203250001e-05, + "loss": 0.4301974105834961, + "mean_token_accuracy": 0.861739870607853, + "num_tokens": 3838629.0, + "step": 1600 + }, + { + "entropy": 0.49970791533589365, + "epoch": 3.6111719605695507, + "grad_norm": 0.7269095182418823, + "learning_rate": 6.101571521996419e-05, + "loss": 0.4372034454345703, + "mean_token_accuracy": 0.8592326313257217, + "num_tokens": 3955462.0, + "step": 1650 + }, + { + "entropy": 0.5049038740992546, + "epoch": 3.7207009857612268, + "grad_norm": 0.6023766398429871, + "learning_rate": 5.98583485209228e-05, + "loss": 0.445910758972168, + "mean_token_accuracy": 0.8579568776488304, + "num_tokens": 4075933.0, + "step": 1700 + }, + { + "entropy": 0.4927579787373543, + "epoch": 3.8302300109529024, + "grad_norm": 0.7742004990577698, + "learning_rate": 5.866883981134422e-05, + "loss": 0.4280668640136719, + "mean_token_accuracy": 0.86241753667593, + "num_tokens": 4192915.0, + "step": 1750 + }, + { + "entropy": 0.5011553263664246, + "epoch": 3.9397590361445785, + "grad_norm": 0.5748111605644226, + "learning_rate": 5.7448923842337736e-05, + "loss": 0.43842597961425783, + "mean_token_accuracy": 0.8605782136321067, + "num_tokens": 4310431.0, + "step": 1800 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4624898036321004, + "eval_loss": 0.590175986289978, + "eval_mean_token_accuracy": 0.8287760076068696, + "eval_num_tokens": 4377896.0, + "eval_runtime": 95.3934, + "eval_samples_per_second": 10.504, + "eval_steps_per_second": 1.321, + "step": 1828 + }, + { + "entropy": 0.4502198097079691, + "epoch": 4.048192771084337, + "grad_norm": 0.7170541882514954, + "learning_rate": 5.620037971023403e-05, + "loss": 0.38744712829589845, + "mean_token_accuracy": 0.8735785065877317, + "num_tokens": 4430089.0, + "step": 1850 + }, + { + "entropy": 0.41583337262272835, + "epoch": 4.157721796276014, + "grad_norm": 0.9511433243751526, + "learning_rate": 5.4925028261993515e-05, + "loss": 0.3562023162841797, + "mean_token_accuracy": 0.881559683084488, + "num_tokens": 4554484.0, + "step": 1900 + }, + { + "entropy": 0.4208242034912109, + "epoch": 4.267250821467689, + "grad_norm": 0.8741114139556885, + "learning_rate": 5.3624729439726544e-05, + "loss": 0.3612668991088867, + "mean_token_accuracy": 0.8802913293242455, + "num_tokens": 4675472.0, + "step": 1950 + }, + { + "entropy": 0.4188448017835617, + "epoch": 4.376779846659365, + "grad_norm": 1.1690106391906738, + "learning_rate": 5.23013795681983e-05, + "loss": 0.3627183151245117, + "mean_token_accuracy": 0.8805234292149544, + "num_tokens": 4793479.0, + "step": 2000 + }, + { + "entropy": 0.42332509815692904, + "epoch": 4.48630887185104, + "grad_norm": 0.8150995969772339, + "learning_rate": 5.095690858927403e-05, + "loss": 0.3626524353027344, + "mean_token_accuracy": 0.879372145831585, + "num_tokens": 4911343.0, + "step": 2050 + }, + { + "entropy": 0.42394075110554696, + "epoch": 4.595837897042716, + "grad_norm": 0.8282762169837952, + "learning_rate": 4.959327724733778e-05, + "loss": 0.3573355865478516, + "mean_token_accuracy": 0.8799301481246948, + "num_tokens": 5028364.0, + "step": 2100 + }, + { + "entropy": 0.4259473057091236, + "epoch": 4.705366922234392, + "grad_norm": 0.7610743045806885, + "learning_rate": 4.8212474229789754e-05, + "loss": 0.3665072631835937, + "mean_token_accuracy": 0.879306109547615, + "num_tokens": 5146995.0, + "step": 2150 + }, + { + "entropy": 0.42407046899199485, + "epoch": 4.814895947426068, + "grad_norm": 0.6995375156402588, + "learning_rate": 4.681651326679193e-05, + "loss": 0.3689637756347656, + "mean_token_accuracy": 0.878270491361618, + "num_tokens": 5263483.0, + "step": 2200 + }, + { + "entropy": 0.41719858527183534, + "epoch": 4.924424972617744, + "grad_norm": 0.8921851515769958, + "learning_rate": 4.5407430194492145e-05, + "loss": 0.36366527557373046, + "mean_token_accuracy": 0.8810832899808884, + "num_tokens": 5387955.0, + "step": 2250 + }, + { + "epoch": 5.0, + "eval_entropy": 0.40321539318750776, + "eval_loss": 0.6344199776649475, + "eval_mean_token_accuracy": 0.8290003571245406, + "eval_num_tokens": 5472370.0, + "eval_runtime": 95.4906, + "eval_samples_per_second": 10.493, + "eval_steps_per_second": 1.32, + "step": 2285 + }, + { + "entropy": 0.39761772337887025, + "epoch": 5.032858707557502, + "grad_norm": 0.8363515734672546, + "learning_rate": 4.3987279986009235e-05, + "loss": 0.3363536834716797, + "mean_token_accuracy": 0.8891537577816935, + "num_tokens": 5508058.0, + "step": 2300 + }, + { + "entropy": 0.3406807939708233, + "epoch": 5.142387732749179, + "grad_norm": 0.9140804409980774, + "learning_rate": 4.2558133754509274e-05, + "loss": 0.2747584533691406, + "mean_token_accuracy": 0.9082898917794228, + "num_tokens": 5625635.0, + "step": 2350 + }, + { + "entropy": 0.34594309888780117, + "epoch": 5.2519167579408546, + "grad_norm": 0.9676663875579834, + "learning_rate": 4.112207573274355e-05, + "loss": 0.2821139907836914, + "mean_token_accuracy": 0.9047485241293907, + "num_tokens": 5741941.0, + "step": 2400 + }, + { + "entropy": 0.3354563079029322, + "epoch": 5.36144578313253, + "grad_norm": 1.1732304096221924, + "learning_rate": 3.968120023345335e-05, + "loss": 0.2757284355163574, + "mean_token_accuracy": 0.905784958600998, + "num_tokens": 5864741.0, + "step": 2450 + }, + { + "entropy": 0.3486690762639046, + "epoch": 5.470974808324206, + "grad_norm": 0.9858622550964355, + "learning_rate": 3.823760859507414e-05, + "loss": 0.28488592147827146, + "mean_token_accuracy": 0.9029894617199897, + "num_tokens": 5984809.0, + "step": 2500 + }, + { + "entropy": 0.35116296328604224, + "epoch": 5.580503833515881, + "grad_norm": 0.8295992016792297, + "learning_rate": 3.679340611719382e-05, + "loss": 0.28818355560302733, + "mean_token_accuracy": 0.9026859793066978, + "num_tokens": 6101316.0, + "step": 2550 + }, + { + "entropy": 0.34762901581823824, + "epoch": 5.690032858707557, + "grad_norm": 1.2317472696304321, + "learning_rate": 3.5350698990234046e-05, + "loss": 0.2834972381591797, + "mean_token_accuracy": 0.9035492998361587, + "num_tokens": 6222300.0, + "step": 2600 + }, + { + "entropy": 0.3509623434394598, + "epoch": 5.7995618838992335, + "grad_norm": 1.1071515083312988, + "learning_rate": 3.391159122383239e-05, + "loss": 0.28417932510375976, + "mean_token_accuracy": 0.9017772257328034, + "num_tokens": 6339706.0, + "step": 2650 + }, + { + "entropy": 0.3327887299656868, + "epoch": 5.909090909090909, + "grad_norm": 0.9410860538482666, + "learning_rate": 3.247818157840487e-05, + "loss": 0.27511814117431643, + "mean_token_accuracy": 0.9069757598638535, + "num_tokens": 6466571.0, + "step": 2700 + }, + { + "epoch": 6.0, + "eval_entropy": 0.37359742701999726, + "eval_loss": 0.6801024079322815, + "eval_mean_token_accuracy": 0.8251904424220796, + "eval_num_tokens": 6566844.0, + "eval_runtime": 95.5582, + "eval_samples_per_second": 10.486, + "eval_steps_per_second": 1.319, + "step": 2742 + }, + { + "entropy": 0.33517336732510367, + "epoch": 6.017524644030668, + "grad_norm": 1.1120449304580688, + "learning_rate": 3.105256050436392e-05, + "loss": 0.2726051139831543, + "mean_token_accuracy": 0.9074887001153195, + "num_tokens": 6588372.0, + "step": 2750 + }, + { + "entropy": 0.2580332762002945, + "epoch": 6.127053669222344, + "grad_norm": 1.1544626951217651, + "learning_rate": 2.9636807093455337e-05, + "loss": 0.1894158172607422, + "mean_token_accuracy": 0.934169539809227, + "num_tokens": 6712562.0, + "step": 2800 + }, + { + "entropy": 0.2648825005441904, + "epoch": 6.23658269441402, + "grad_norm": 1.2022897005081177, + "learning_rate": 2.823298604666056e-05, + "loss": 0.19440870285034179, + "mean_token_accuracy": 0.9334753274917602, + "num_tokens": 6832843.0, + "step": 2850 + }, + { + "entropy": 0.27026796594262126, + "epoch": 6.3461117196056955, + "grad_norm": 1.0150978565216064, + "learning_rate": 2.6843144663086045e-05, + "loss": 0.19671850204467772, + "mean_token_accuracy": 0.9316870296001434, + "num_tokens": 6952871.0, + "step": 2900 + }, + { + "entropy": 0.2634010723978281, + "epoch": 6.455640744797371, + "grad_norm": 1.430558681488037, + "learning_rate": 2.546930985423105e-05, + "loss": 0.19457483291625977, + "mean_token_accuracy": 0.9327938884496689, + "num_tokens": 7072067.0, + "step": 2950 + }, + { + "entropy": 0.2702385004609823, + "epoch": 6.565169769989047, + "grad_norm": 1.0556726455688477, + "learning_rate": 2.4113485187988342e-05, + "loss": 0.19928800582885742, + "mean_token_accuracy": 0.9298818710446358, + "num_tokens": 7187280.0, + "step": 3000 + }, + { + "entropy": 0.2702864905446768, + "epoch": 6.674698795180722, + "grad_norm": 1.1694364547729492, + "learning_rate": 2.2777647966688595e-05, + "loss": 0.20284730911254883, + "mean_token_accuracy": 0.9301017987728118, + "num_tokens": 7305186.0, + "step": 3050 + }, + { + "entropy": 0.2645207424461842, + "epoch": 6.784227820372399, + "grad_norm": 1.2039296627044678, + "learning_rate": 2.146374634344989e-05, + "loss": 0.1961233139038086, + "mean_token_accuracy": 0.9320311924815178, + "num_tokens": 7427803.0, + "step": 3100 + }, + { + "entropy": 0.2703990802913904, + "epoch": 6.8937568455640745, + "grad_norm": 1.122693657875061, + "learning_rate": 2.01736964810376e-05, + "loss": 0.19937246322631835, + "mean_token_accuracy": 0.9296649679541588, + "num_tokens": 7543735.0, + "step": 3150 + }, + { + "epoch": 7.0, + "eval_entropy": 0.31011983941471766, + "eval_loss": 0.7977674007415771, + "eval_mean_token_accuracy": 0.8200774944963909, + "eval_num_tokens": 7661318.0, + "eval_runtime": 95.5905, + "eval_samples_per_second": 10.482, + "eval_steps_per_second": 1.318, + "step": 3199 + } + ], + "logging_steps": 50, + "max_steps": 4570, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.268827488643707e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/README.md b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/adapter_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a72a4c543bf8671cb7019360179c8cabb6e5d6c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.021223473447411947, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "gate_proj", + "v_proj", + "up_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/chat_template.jinja b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/tokenizer_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/trainer_state.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..38ce1f1ef51f3a92239d594a59917d3f2bf0fa89 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-3656/trainer_state.json @@ -0,0 +1,852 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3656, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.002540482878685, + "epoch": 0.10952902519167579, + "grad_norm": 0.5552461743354797, + "learning_rate": 8.109930924389919e-06, + "loss": 1.9067156982421876, + "mean_token_accuracy": 0.6026969534158707, + "num_tokens": 119060.0, + "step": 50 + }, + { + "entropy": 1.3068562260270118, + "epoch": 0.21905805038335158, + "grad_norm": 1.0465189218521118, + "learning_rate": 1.6385370643155144e-05, + "loss": 1.1655167388916015, + "mean_token_accuracy": 0.7124340075254441, + "num_tokens": 242713.0, + "step": 100 + }, + { + "entropy": 0.8696084088087082, + "epoch": 0.32858707557502737, + "grad_norm": 0.6686795949935913, + "learning_rate": 2.4660810361920367e-05, + "loss": 0.8031976318359375, + "mean_token_accuracy": 0.7754033240675926, + "num_tokens": 360160.0, + "step": 150 + }, + { + "entropy": 0.7853927117586136, + "epoch": 0.43811610076670315, + "grad_norm": 0.9354436993598938, + "learning_rate": 3.293625008068559e-05, + "loss": 0.7226885986328125, + "mean_token_accuracy": 0.791197614967823, + "num_tokens": 485403.0, + "step": 200 + }, + { + "entropy": 0.7382483741641045, + "epoch": 0.547645125958379, + "grad_norm": 0.7535398006439209, + "learning_rate": 4.121168979945081e-05, + "loss": 0.6755471801757813, + "mean_token_accuracy": 0.8054334259033203, + "num_tokens": 601578.0, + "step": 250 + }, + { + "entropy": 0.7218390592932701, + "epoch": 0.6571741511500547, + "grad_norm": 0.8160243034362793, + "learning_rate": 4.948712951821604e-05, + "loss": 0.6614183044433594, + "mean_token_accuracy": 0.8060630604624748, + "num_tokens": 713917.0, + "step": 300 + }, + { + "entropy": 0.7012957927584648, + "epoch": 0.7667031763417306, + "grad_norm": 0.7985761165618896, + "learning_rate": 5.776256923698126e-05, + "loss": 0.6415711975097657, + "mean_token_accuracy": 0.8100408402085304, + "num_tokens": 832995.0, + "step": 350 + }, + { + "entropy": 0.6763210469484329, + "epoch": 0.8762322015334063, + "grad_norm": 0.6829700469970703, + "learning_rate": 6.603800895574648e-05, + "loss": 0.6193617248535156, + "mean_token_accuracy": 0.8140933158993721, + "num_tokens": 959304.0, + "step": 400 + }, + { + "entropy": 0.6874829810857773, + "epoch": 0.9857612267250822, + "grad_norm": 0.6983632445335388, + "learning_rate": 7.43134486745117e-05, + "loss": 0.6262834167480469, + "mean_token_accuracy": 0.8163631609082223, + "num_tokens": 1078880.0, + "step": 450 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6085402492492918, + "eval_loss": 0.6388216018676758, + "eval_mean_token_accuracy": 0.8123693073552752, + "eval_num_tokens": 1094474.0, + "eval_runtime": 95.9846, + "eval_samples_per_second": 10.439, + "eval_steps_per_second": 1.313, + "step": 457 + }, + { + "entropy": 0.6471366012337232, + "epoch": 1.0941949616648412, + "grad_norm": 0.7878272533416748, + "learning_rate": 7.561806001041411e-05, + "loss": 0.589820671081543, + "mean_token_accuracy": 0.8218218798589225, + "num_tokens": 1190065.0, + "step": 500 + }, + { + "entropy": 0.6404865515232087, + "epoch": 1.203723986856517, + "grad_norm": 0.7976964712142944, + "learning_rate": 7.554418144822605e-05, + "loss": 0.5836894607543945, + "mean_token_accuracy": 0.8222010856866837, + "num_tokens": 1313079.0, + "step": 550 + }, + { + "entropy": 0.6355892798304558, + "epoch": 1.3132530120481927, + "grad_norm": 0.7428044080734253, + "learning_rate": 7.541528503116934e-05, + "loss": 0.577253189086914, + "mean_token_accuracy": 0.8260676205158234, + "num_tokens": 1428305.0, + "step": 600 + }, + { + "entropy": 0.6170084626972675, + "epoch": 1.4227820372398685, + "grad_norm": 0.6792078614234924, + "learning_rate": 7.523155873870194e-05, + "loss": 0.561871337890625, + "mean_token_accuracy": 0.8307154527306557, + "num_tokens": 1551971.0, + "step": 650 + }, + { + "entropy": 0.6222476975619793, + "epoch": 1.5323110624315444, + "grad_norm": 0.6532447934150696, + "learning_rate": 7.499327051286336e-05, + "loss": 0.5612493515014648, + "mean_token_accuracy": 0.8296262130141259, + "num_tokens": 1673401.0, + "step": 700 + }, + { + "entropy": 0.6251883202791214, + "epoch": 1.6418400876232202, + "grad_norm": 0.7107782959938049, + "learning_rate": 7.47007678675144e-05, + "loss": 0.565279769897461, + "mean_token_accuracy": 0.8299687370657921, + "num_tokens": 1788813.0, + "step": 750 + }, + { + "entropy": 0.6177104935050011, + "epoch": 1.751369112814896, + "grad_norm": 0.7850057482719421, + "learning_rate": 7.435447738153122e-05, + "loss": 0.5564990234375, + "mean_token_accuracy": 0.8291967037320137, + "num_tokens": 1911556.0, + "step": 800 + }, + { + "entropy": 0.6029443763196468, + "epoch": 1.8608981380065717, + "grad_norm": 0.7116957306861877, + "learning_rate": 7.395490407669285e-05, + "loss": 0.5501844406127929, + "mean_token_accuracy": 0.8335196697711944, + "num_tokens": 2029129.0, + "step": 850 + }, + { + "entropy": 0.6042105440795421, + "epoch": 1.9704271631982475, + "grad_norm": 0.6712159514427185, + "learning_rate": 7.350263068116955e-05, + "loss": 0.5517086029052735, + "mean_token_accuracy": 0.8321291375160217, + "num_tokens": 2155392.0, + "step": 900 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5401396741942753, + "eval_loss": 0.5797445774078369, + "eval_mean_token_accuracy": 0.8243241475688087, + "eval_num_tokens": 2188948.0, + "eval_runtime": 95.5407, + "eval_samples_per_second": 10.488, + "eval_steps_per_second": 1.319, + "step": 914 + }, + { + "entropy": 0.5675096463675451, + "epoch": 2.078860898138007, + "grad_norm": 0.5713562369346619, + "learning_rate": 7.299831677968588e-05, + "loss": 0.5120392227172852, + "mean_token_accuracy": 0.8414448993374603, + "num_tokens": 2277503.0, + "step": 950 + }, + { + "entropy": 0.5500032117962838, + "epoch": 2.1883899233296824, + "grad_norm": 0.5951120257377625, + "learning_rate": 7.244269785159817e-05, + "loss": 0.49384498596191406, + "mean_token_accuracy": 0.8470860269665718, + "num_tokens": 2392968.0, + "step": 1000 + }, + { + "entropy": 0.5683873899281024, + "epoch": 2.297918948521358, + "grad_norm": 0.6816521286964417, + "learning_rate": 7.183658419828891e-05, + "loss": 0.5088459014892578, + "mean_token_accuracy": 0.8413213565945625, + "num_tokens": 2507108.0, + "step": 1050 + }, + { + "entropy": 0.5481136417388917, + "epoch": 2.407447973713034, + "grad_norm": 0.6417970657348633, + "learning_rate": 7.118085976144257e-05, + "loss": 0.49456378936767575, + "mean_token_accuracy": 0.8468478980660439, + "num_tokens": 2633824.0, + "step": 1100 + }, + { + "entropy": 0.5413966289162636, + "epoch": 2.5169769989047097, + "grad_norm": 0.631996214389801, + "learning_rate": 7.047648083392619e-05, + "loss": 0.49154373168945314, + "mean_token_accuracy": 0.8461908429861069, + "num_tokens": 2753131.0, + "step": 1150 + }, + { + "entropy": 0.5583206915855408, + "epoch": 2.6265060240963853, + "grad_norm": 0.7409902215003967, + "learning_rate": 6.972447466515462e-05, + "loss": 0.4927285385131836, + "mean_token_accuracy": 0.8451325806975365, + "num_tokens": 2865112.0, + "step": 1200 + }, + { + "entropy": 0.556266717761755, + "epoch": 2.7360350492880614, + "grad_norm": 0.5275787115097046, + "learning_rate": 6.892593796297452e-05, + "loss": 0.499769401550293, + "mean_token_accuracy": 0.8451313543319702, + "num_tokens": 2980972.0, + "step": 1250 + }, + { + "entropy": 0.5362468618154526, + "epoch": 2.845564074479737, + "grad_norm": 0.5450661182403564, + "learning_rate": 6.808203529425189e-05, + "loss": 0.4860528945922852, + "mean_token_accuracy": 0.8479894894361496, + "num_tokens": 3108708.0, + "step": 1300 + }, + { + "entropy": 0.5334719524532556, + "epoch": 2.955093099671413, + "grad_norm": 0.4825150966644287, + "learning_rate": 6.719399738649542e-05, + "loss": 0.48385780334472656, + "mean_token_accuracy": 0.8489498183131218, + "num_tokens": 3232991.0, + "step": 1350 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5111288797287714, + "eval_loss": 0.5738435387611389, + "eval_mean_token_accuracy": 0.8259959235077813, + "eval_num_tokens": 3283422.0, + "eval_runtime": 95.6002, + "eval_samples_per_second": 10.481, + "eval_steps_per_second": 1.318, + "step": 1371 + }, + { + "entropy": 0.5024978819519582, + "epoch": 3.063526834611172, + "grad_norm": 0.6228470802307129, + "learning_rate": 6.626311933299292e-05, + "loss": 0.4451956939697266, + "mean_token_accuracy": 0.8579568441468056, + "num_tokens": 3352672.0, + "step": 1400 + }, + { + "entropy": 0.49351966604590414, + "epoch": 3.1730558598028478, + "grad_norm": 0.648960530757904, + "learning_rate": 6.529075870407823e-05, + "loss": 0.4324279022216797, + "mean_token_accuracy": 0.8607994091510772, + "num_tokens": 3472463.0, + "step": 1450 + }, + { + "entropy": 0.48342301592230796, + "epoch": 3.2825848849945234, + "grad_norm": 0.8443573713302612, + "learning_rate": 6.427833356728302e-05, + "loss": 0.4237791442871094, + "mean_token_accuracy": 0.8643713328242302, + "num_tokens": 3593837.0, + "step": 1500 + }, + { + "entropy": 0.4779162485897541, + "epoch": 3.3921139101861995, + "grad_norm": 0.7882111072540283, + "learning_rate": 6.32273204192609e-05, + "loss": 0.42386363983154296, + "mean_token_accuracy": 0.8635856115818024, + "num_tokens": 3716058.0, + "step": 1550 + }, + { + "entropy": 0.48870255261659623, + "epoch": 3.501642935377875, + "grad_norm": 0.7637454867362976, + "learning_rate": 6.213925203250001e-05, + "loss": 0.4301974105834961, + "mean_token_accuracy": 0.861739870607853, + "num_tokens": 3838629.0, + "step": 1600 + }, + { + "entropy": 0.49970791533589365, + "epoch": 3.6111719605695507, + "grad_norm": 0.7269095182418823, + "learning_rate": 6.101571521996419e-05, + "loss": 0.4372034454345703, + "mean_token_accuracy": 0.8592326313257217, + "num_tokens": 3955462.0, + "step": 1650 + }, + { + "entropy": 0.5049038740992546, + "epoch": 3.7207009857612268, + "grad_norm": 0.6023766398429871, + "learning_rate": 5.98583485209228e-05, + "loss": 0.445910758972168, + "mean_token_accuracy": 0.8579568776488304, + "num_tokens": 4075933.0, + "step": 1700 + }, + { + "entropy": 0.4927579787373543, + "epoch": 3.8302300109529024, + "grad_norm": 0.7742004990577698, + "learning_rate": 5.866883981134422e-05, + "loss": 0.4280668640136719, + "mean_token_accuracy": 0.86241753667593, + "num_tokens": 4192915.0, + "step": 1750 + }, + { + "entropy": 0.5011553263664246, + "epoch": 3.9397590361445785, + "grad_norm": 0.5748111605644226, + "learning_rate": 5.7448923842337736e-05, + "loss": 0.43842597961425783, + "mean_token_accuracy": 0.8605782136321067, + "num_tokens": 4310431.0, + "step": 1800 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4624898036321004, + "eval_loss": 0.590175986289978, + "eval_mean_token_accuracy": 0.8287760076068696, + "eval_num_tokens": 4377896.0, + "eval_runtime": 95.3934, + "eval_samples_per_second": 10.504, + "eval_steps_per_second": 1.321, + "step": 1828 + }, + { + "entropy": 0.4502198097079691, + "epoch": 4.048192771084337, + "grad_norm": 0.7170541882514954, + "learning_rate": 5.620037971023403e-05, + "loss": 0.38744712829589845, + "mean_token_accuracy": 0.8735785065877317, + "num_tokens": 4430089.0, + "step": 1850 + }, + { + "entropy": 0.41583337262272835, + "epoch": 4.157721796276014, + "grad_norm": 0.9511433243751526, + "learning_rate": 5.4925028261993515e-05, + "loss": 0.3562023162841797, + "mean_token_accuracy": 0.881559683084488, + "num_tokens": 4554484.0, + "step": 1900 + }, + { + "entropy": 0.4208242034912109, + "epoch": 4.267250821467689, + "grad_norm": 0.8741114139556885, + "learning_rate": 5.3624729439726544e-05, + "loss": 0.3612668991088867, + "mean_token_accuracy": 0.8802913293242455, + "num_tokens": 4675472.0, + "step": 1950 + }, + { + "entropy": 0.4188448017835617, + "epoch": 4.376779846659365, + "grad_norm": 1.1690106391906738, + "learning_rate": 5.23013795681983e-05, + "loss": 0.3627183151245117, + "mean_token_accuracy": 0.8805234292149544, + "num_tokens": 4793479.0, + "step": 2000 + }, + { + "entropy": 0.42332509815692904, + "epoch": 4.48630887185104, + "grad_norm": 0.8150995969772339, + "learning_rate": 5.095690858927403e-05, + "loss": 0.3626524353027344, + "mean_token_accuracy": 0.879372145831585, + "num_tokens": 4911343.0, + "step": 2050 + }, + { + "entropy": 0.42394075110554696, + "epoch": 4.595837897042716, + "grad_norm": 0.8282762169837952, + "learning_rate": 4.959327724733778e-05, + "loss": 0.3573355865478516, + "mean_token_accuracy": 0.8799301481246948, + "num_tokens": 5028364.0, + "step": 2100 + }, + { + "entropy": 0.4259473057091236, + "epoch": 4.705366922234392, + "grad_norm": 0.7610743045806885, + "learning_rate": 4.8212474229789754e-05, + "loss": 0.3665072631835937, + "mean_token_accuracy": 0.879306109547615, + "num_tokens": 5146995.0, + "step": 2150 + }, + { + "entropy": 0.42407046899199485, + "epoch": 4.814895947426068, + "grad_norm": 0.6995375156402588, + "learning_rate": 4.681651326679193e-05, + "loss": 0.3689637756347656, + "mean_token_accuracy": 0.878270491361618, + "num_tokens": 5263483.0, + "step": 2200 + }, + { + "entropy": 0.41719858527183534, + "epoch": 4.924424972617744, + "grad_norm": 0.8921851515769958, + "learning_rate": 4.5407430194492145e-05, + "loss": 0.36366527557373046, + "mean_token_accuracy": 0.8810832899808884, + "num_tokens": 5387955.0, + "step": 2250 + }, + { + "epoch": 5.0, + "eval_entropy": 0.40321539318750776, + "eval_loss": 0.6344199776649475, + "eval_mean_token_accuracy": 0.8290003571245406, + "eval_num_tokens": 5472370.0, + "eval_runtime": 95.4906, + "eval_samples_per_second": 10.493, + "eval_steps_per_second": 1.32, + "step": 2285 + }, + { + "entropy": 0.39761772337887025, + "epoch": 5.032858707557502, + "grad_norm": 0.8363515734672546, + "learning_rate": 4.3987279986009235e-05, + "loss": 0.3363536834716797, + "mean_token_accuracy": 0.8891537577816935, + "num_tokens": 5508058.0, + "step": 2300 + }, + { + "entropy": 0.3406807939708233, + "epoch": 5.142387732749179, + "grad_norm": 0.9140804409980774, + "learning_rate": 4.2558133754509274e-05, + "loss": 0.2747584533691406, + "mean_token_accuracy": 0.9082898917794228, + "num_tokens": 5625635.0, + "step": 2350 + }, + { + "entropy": 0.34594309888780117, + "epoch": 5.2519167579408546, + "grad_norm": 0.9676663875579834, + "learning_rate": 4.112207573274355e-05, + "loss": 0.2821139907836914, + "mean_token_accuracy": 0.9047485241293907, + "num_tokens": 5741941.0, + "step": 2400 + }, + { + "entropy": 0.3354563079029322, + "epoch": 5.36144578313253, + "grad_norm": 1.1732304096221924, + "learning_rate": 3.968120023345335e-05, + "loss": 0.2757284355163574, + "mean_token_accuracy": 0.905784958600998, + "num_tokens": 5864741.0, + "step": 2450 + }, + { + "entropy": 0.3486690762639046, + "epoch": 5.470974808324206, + "grad_norm": 0.9858622550964355, + "learning_rate": 3.823760859507414e-05, + "loss": 0.28488592147827146, + "mean_token_accuracy": 0.9029894617199897, + "num_tokens": 5984809.0, + "step": 2500 + }, + { + "entropy": 0.35116296328604224, + "epoch": 5.580503833515881, + "grad_norm": 0.8295992016792297, + "learning_rate": 3.679340611719382e-05, + "loss": 0.28818355560302733, + "mean_token_accuracy": 0.9026859793066978, + "num_tokens": 6101316.0, + "step": 2550 + }, + { + "entropy": 0.34762901581823824, + "epoch": 5.690032858707557, + "grad_norm": 1.2317472696304321, + "learning_rate": 3.5350698990234046e-05, + "loss": 0.2834972381591797, + "mean_token_accuracy": 0.9035492998361587, + "num_tokens": 6222300.0, + "step": 2600 + }, + { + "entropy": 0.3509623434394598, + "epoch": 5.7995618838992335, + "grad_norm": 1.1071515083312988, + "learning_rate": 3.391159122383239e-05, + "loss": 0.28417932510375976, + "mean_token_accuracy": 0.9017772257328034, + "num_tokens": 6339706.0, + "step": 2650 + }, + { + "entropy": 0.3327887299656868, + "epoch": 5.909090909090909, + "grad_norm": 0.9410860538482666, + "learning_rate": 3.247818157840487e-05, + "loss": 0.27511814117431643, + "mean_token_accuracy": 0.9069757598638535, + "num_tokens": 6466571.0, + "step": 2700 + }, + { + "epoch": 6.0, + "eval_entropy": 0.37359742701999726, + "eval_loss": 0.6801024079322815, + "eval_mean_token_accuracy": 0.8251904424220796, + "eval_num_tokens": 6566844.0, + "eval_runtime": 95.5582, + "eval_samples_per_second": 10.486, + "eval_steps_per_second": 1.319, + "step": 2742 + }, + { + "entropy": 0.33517336732510367, + "epoch": 6.017524644030668, + "grad_norm": 1.1120449304580688, + "learning_rate": 3.105256050436392e-05, + "loss": 0.2726051139831543, + "mean_token_accuracy": 0.9074887001153195, + "num_tokens": 6588372.0, + "step": 2750 + }, + { + "entropy": 0.2580332762002945, + "epoch": 6.127053669222344, + "grad_norm": 1.1544626951217651, + "learning_rate": 2.9636807093455337e-05, + "loss": 0.1894158172607422, + "mean_token_accuracy": 0.934169539809227, + "num_tokens": 6712562.0, + "step": 2800 + }, + { + "entropy": 0.2648825005441904, + "epoch": 6.23658269441402, + "grad_norm": 1.2022897005081177, + "learning_rate": 2.823298604666056e-05, + "loss": 0.19440870285034179, + "mean_token_accuracy": 0.9334753274917602, + "num_tokens": 6832843.0, + "step": 2850 + }, + { + "entropy": 0.27026796594262126, + "epoch": 6.3461117196056955, + "grad_norm": 1.0150978565216064, + "learning_rate": 2.6843144663086045e-05, + "loss": 0.19671850204467772, + "mean_token_accuracy": 0.9316870296001434, + "num_tokens": 6952871.0, + "step": 2900 + }, + { + "entropy": 0.2634010723978281, + "epoch": 6.455640744797371, + "grad_norm": 1.430558681488037, + "learning_rate": 2.546930985423105e-05, + "loss": 0.19457483291625977, + "mean_token_accuracy": 0.9327938884496689, + "num_tokens": 7072067.0, + "step": 2950 + }, + { + "entropy": 0.2702385004609823, + "epoch": 6.565169769989047, + "grad_norm": 1.0556726455688477, + "learning_rate": 2.4113485187988342e-05, + "loss": 0.19928800582885742, + "mean_token_accuracy": 0.9298818710446358, + "num_tokens": 7187280.0, + "step": 3000 + }, + { + "entropy": 0.2702864905446768, + "epoch": 6.674698795180722, + "grad_norm": 1.1694364547729492, + "learning_rate": 2.2777647966688595e-05, + "loss": 0.20284730911254883, + "mean_token_accuracy": 0.9301017987728118, + "num_tokens": 7305186.0, + "step": 3050 + }, + { + "entropy": 0.2645207424461842, + "epoch": 6.784227820372399, + "grad_norm": 1.2039296627044678, + "learning_rate": 2.146374634344989e-05, + "loss": 0.1961233139038086, + "mean_token_accuracy": 0.9320311924815178, + "num_tokens": 7427803.0, + "step": 3100 + }, + { + "entropy": 0.2703990802913904, + "epoch": 6.8937568455640745, + "grad_norm": 1.122693657875061, + "learning_rate": 2.01736964810376e-05, + "loss": 0.19937246322631835, + "mean_token_accuracy": 0.9296649679541588, + "num_tokens": 7543735.0, + "step": 3150 + }, + { + "epoch": 7.0, + "eval_entropy": 0.31011983941471766, + "eval_loss": 0.7977674007415771, + "eval_mean_token_accuracy": 0.8200774944963909, + "eval_num_tokens": 7661318.0, + "eval_runtime": 95.5905, + "eval_samples_per_second": 10.482, + "eval_steps_per_second": 1.318, + "step": 3199 + }, + { + "entropy": 0.27026369738759415, + "epoch": 7.002190580503833, + "grad_norm": 0.8098449110984802, + "learning_rate": 1.8909379757378543e-05, + "loss": 0.19773469924926756, + "mean_token_accuracy": 0.9312729013688636, + "num_tokens": 7663078.0, + "step": 3200 + }, + { + "entropy": 0.21044189006090164, + "epoch": 7.11171960569551, + "grad_norm": 1.2920125722885132, + "learning_rate": 1.7672640021804126e-05, + "loss": 0.13121842384338378, + "mean_token_accuracy": 0.954786310493946, + "num_tokens": 7782363.0, + "step": 3250 + }, + { + "entropy": 0.2033486431464553, + "epoch": 7.221248630887185, + "grad_norm": 1.1921308040618896, + "learning_rate": 1.6465280906024633e-05, + "loss": 0.13039697647094728, + "mean_token_accuracy": 0.9554795321822166, + "num_tokens": 7899912.0, + "step": 3300 + }, + { + "entropy": 0.2082062489911914, + "epoch": 7.330777656078861, + "grad_norm": 1.1978024244308472, + "learning_rate": 1.5289063193755705e-05, + "loss": 0.13286386489868163, + "mean_token_accuracy": 0.9534762743115425, + "num_tokens": 8019370.0, + "step": 3350 + }, + { + "entropy": 0.2054990328848362, + "epoch": 7.4403066812705365, + "grad_norm": 1.1366864442825317, + "learning_rate": 1.4145702252833402e-05, + "loss": 0.1314125633239746, + "mean_token_accuracy": 0.9538632571697235, + "num_tokens": 8142751.0, + "step": 3400 + }, + { + "entropy": 0.20359413929283618, + "epoch": 7.549835706462212, + "grad_norm": 1.2026054859161377, + "learning_rate": 1.3036865533562794e-05, + "loss": 0.1330260467529297, + "mean_token_accuracy": 0.9539996239542962, + "num_tokens": 8260948.0, + "step": 3450 + }, + { + "entropy": 0.21044832307845354, + "epoch": 7.659364731653889, + "grad_norm": 1.3045598268508911, + "learning_rate": 1.1964170136948073e-05, + "loss": 0.13348891258239745, + "mean_token_accuracy": 0.9526105988025665, + "num_tokens": 8378334.0, + "step": 3500 + }, + { + "entropy": 0.20063834685832263, + "epoch": 7.768893756845564, + "grad_norm": 1.001644253730774, + "learning_rate": 1.0929180456351225e-05, + "loss": 0.12778966903686523, + "mean_token_accuracy": 0.955841820538044, + "num_tokens": 8503618.0, + "step": 3550 + }, + { + "entropy": 0.20864798523485661, + "epoch": 7.87842278203724, + "grad_norm": 1.0793193578720093, + "learning_rate": 9.933405896017934e-06, + "loss": 0.13261634826660157, + "mean_token_accuracy": 0.9527471828460693, + "num_tokens": 8621388.0, + "step": 3600 + }, + { + "entropy": 0.2030112524703145, + "epoch": 7.9879518072289155, + "grad_norm": 0.9351119995117188, + "learning_rate": 8.978298669798568e-06, + "loss": 0.13032340049743651, + "mean_token_accuracy": 0.953511869609356, + "num_tokens": 8743662.0, + "step": 3650 + }, + { + "epoch": 8.0, + "eval_entropy": 0.27168176760749213, + "eval_loss": 0.9082825779914856, + "eval_mean_token_accuracy": 0.8165905882441808, + "eval_num_tokens": 8755792.0, + "eval_runtime": 95.6338, + "eval_samples_per_second": 10.477, + "eval_steps_per_second": 1.318, + "step": 3656 + } + ], + "logging_steps": 50, + "max_steps": 4570, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.4504787468625306e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/README.md b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/adapter_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a72a4c543bf8671cb7019360179c8cabb6e5d6c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.021223473447411947, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "gate_proj", + "v_proj", + "up_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/chat_template.jinja b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/tokenizer_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/trainer_state.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..5ecb14add85647985626ec9ffe48851d4147c7e6 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4113/trainer_state.json @@ -0,0 +1,953 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 4113, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.002540482878685, + "epoch": 0.10952902519167579, + "grad_norm": 0.5552461743354797, + "learning_rate": 8.109930924389919e-06, + "loss": 1.9067156982421876, + "mean_token_accuracy": 0.6026969534158707, + "num_tokens": 119060.0, + "step": 50 + }, + { + "entropy": 1.3068562260270118, + "epoch": 0.21905805038335158, + "grad_norm": 1.0465189218521118, + "learning_rate": 1.6385370643155144e-05, + "loss": 1.1655167388916015, + "mean_token_accuracy": 0.7124340075254441, + "num_tokens": 242713.0, + "step": 100 + }, + { + "entropy": 0.8696084088087082, + "epoch": 0.32858707557502737, + "grad_norm": 0.6686795949935913, + "learning_rate": 2.4660810361920367e-05, + "loss": 0.8031976318359375, + "mean_token_accuracy": 0.7754033240675926, + "num_tokens": 360160.0, + "step": 150 + }, + { + "entropy": 0.7853927117586136, + "epoch": 0.43811610076670315, + "grad_norm": 0.9354436993598938, + "learning_rate": 3.293625008068559e-05, + "loss": 0.7226885986328125, + "mean_token_accuracy": 0.791197614967823, + "num_tokens": 485403.0, + "step": 200 + }, + { + "entropy": 0.7382483741641045, + "epoch": 0.547645125958379, + "grad_norm": 0.7535398006439209, + "learning_rate": 4.121168979945081e-05, + "loss": 0.6755471801757813, + "mean_token_accuracy": 0.8054334259033203, + "num_tokens": 601578.0, + "step": 250 + }, + { + "entropy": 0.7218390592932701, + "epoch": 0.6571741511500547, + "grad_norm": 0.8160243034362793, + "learning_rate": 4.948712951821604e-05, + "loss": 0.6614183044433594, + "mean_token_accuracy": 0.8060630604624748, + "num_tokens": 713917.0, + "step": 300 + }, + { + "entropy": 0.7012957927584648, + "epoch": 0.7667031763417306, + "grad_norm": 0.7985761165618896, + "learning_rate": 5.776256923698126e-05, + "loss": 0.6415711975097657, + "mean_token_accuracy": 0.8100408402085304, + "num_tokens": 832995.0, + "step": 350 + }, + { + "entropy": 0.6763210469484329, + "epoch": 0.8762322015334063, + "grad_norm": 0.6829700469970703, + "learning_rate": 6.603800895574648e-05, + "loss": 0.6193617248535156, + "mean_token_accuracy": 0.8140933158993721, + "num_tokens": 959304.0, + "step": 400 + }, + { + "entropy": 0.6874829810857773, + "epoch": 0.9857612267250822, + "grad_norm": 0.6983632445335388, + "learning_rate": 7.43134486745117e-05, + "loss": 0.6262834167480469, + "mean_token_accuracy": 0.8163631609082223, + "num_tokens": 1078880.0, + "step": 450 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6085402492492918, + "eval_loss": 0.6388216018676758, + "eval_mean_token_accuracy": 0.8123693073552752, + "eval_num_tokens": 1094474.0, + "eval_runtime": 95.9846, + "eval_samples_per_second": 10.439, + "eval_steps_per_second": 1.313, + "step": 457 + }, + { + "entropy": 0.6471366012337232, + "epoch": 1.0941949616648412, + "grad_norm": 0.7878272533416748, + "learning_rate": 7.561806001041411e-05, + "loss": 0.589820671081543, + "mean_token_accuracy": 0.8218218798589225, + "num_tokens": 1190065.0, + "step": 500 + }, + { + "entropy": 0.6404865515232087, + "epoch": 1.203723986856517, + "grad_norm": 0.7976964712142944, + "learning_rate": 7.554418144822605e-05, + "loss": 0.5836894607543945, + "mean_token_accuracy": 0.8222010856866837, + "num_tokens": 1313079.0, + "step": 550 + }, + { + "entropy": 0.6355892798304558, + "epoch": 1.3132530120481927, + "grad_norm": 0.7428044080734253, + "learning_rate": 7.541528503116934e-05, + "loss": 0.577253189086914, + "mean_token_accuracy": 0.8260676205158234, + "num_tokens": 1428305.0, + "step": 600 + }, + { + "entropy": 0.6170084626972675, + "epoch": 1.4227820372398685, + "grad_norm": 0.6792078614234924, + "learning_rate": 7.523155873870194e-05, + "loss": 0.561871337890625, + "mean_token_accuracy": 0.8307154527306557, + "num_tokens": 1551971.0, + "step": 650 + }, + { + "entropy": 0.6222476975619793, + "epoch": 1.5323110624315444, + "grad_norm": 0.6532447934150696, + "learning_rate": 7.499327051286336e-05, + "loss": 0.5612493515014648, + "mean_token_accuracy": 0.8296262130141259, + "num_tokens": 1673401.0, + "step": 700 + }, + { + "entropy": 0.6251883202791214, + "epoch": 1.6418400876232202, + "grad_norm": 0.7107782959938049, + "learning_rate": 7.47007678675144e-05, + "loss": 0.565279769897461, + "mean_token_accuracy": 0.8299687370657921, + "num_tokens": 1788813.0, + "step": 750 + }, + { + "entropy": 0.6177104935050011, + "epoch": 1.751369112814896, + "grad_norm": 0.7850057482719421, + "learning_rate": 7.435447738153122e-05, + "loss": 0.5564990234375, + "mean_token_accuracy": 0.8291967037320137, + "num_tokens": 1911556.0, + "step": 800 + }, + { + "entropy": 0.6029443763196468, + "epoch": 1.8608981380065717, + "grad_norm": 0.7116957306861877, + "learning_rate": 7.395490407669285e-05, + "loss": 0.5501844406127929, + "mean_token_accuracy": 0.8335196697711944, + "num_tokens": 2029129.0, + "step": 850 + }, + { + "entropy": 0.6042105440795421, + "epoch": 1.9704271631982475, + "grad_norm": 0.6712159514427185, + "learning_rate": 7.350263068116955e-05, + "loss": 0.5517086029052735, + "mean_token_accuracy": 0.8321291375160217, + "num_tokens": 2155392.0, + "step": 900 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5401396741942753, + "eval_loss": 0.5797445774078369, + "eval_mean_token_accuracy": 0.8243241475688087, + "eval_num_tokens": 2188948.0, + "eval_runtime": 95.5407, + "eval_samples_per_second": 10.488, + "eval_steps_per_second": 1.319, + "step": 914 + }, + { + "entropy": 0.5675096463675451, + "epoch": 2.078860898138007, + "grad_norm": 0.5713562369346619, + "learning_rate": 7.299831677968588e-05, + "loss": 0.5120392227172852, + "mean_token_accuracy": 0.8414448993374603, + "num_tokens": 2277503.0, + "step": 950 + }, + { + "entropy": 0.5500032117962838, + "epoch": 2.1883899233296824, + "grad_norm": 0.5951120257377625, + "learning_rate": 7.244269785159817e-05, + "loss": 0.49384498596191406, + "mean_token_accuracy": 0.8470860269665718, + "num_tokens": 2392968.0, + "step": 1000 + }, + { + "entropy": 0.5683873899281024, + "epoch": 2.297918948521358, + "grad_norm": 0.6816521286964417, + "learning_rate": 7.183658419828891e-05, + "loss": 0.5088459014892578, + "mean_token_accuracy": 0.8413213565945625, + "num_tokens": 2507108.0, + "step": 1050 + }, + { + "entropy": 0.5481136417388917, + "epoch": 2.407447973713034, + "grad_norm": 0.6417970657348633, + "learning_rate": 7.118085976144257e-05, + "loss": 0.49456378936767575, + "mean_token_accuracy": 0.8468478980660439, + "num_tokens": 2633824.0, + "step": 1100 + }, + { + "entropy": 0.5413966289162636, + "epoch": 2.5169769989047097, + "grad_norm": 0.631996214389801, + "learning_rate": 7.047648083392619e-05, + "loss": 0.49154373168945314, + "mean_token_accuracy": 0.8461908429861069, + "num_tokens": 2753131.0, + "step": 1150 + }, + { + "entropy": 0.5583206915855408, + "epoch": 2.6265060240963853, + "grad_norm": 0.7409902215003967, + "learning_rate": 6.972447466515462e-05, + "loss": 0.4927285385131836, + "mean_token_accuracy": 0.8451325806975365, + "num_tokens": 2865112.0, + "step": 1200 + }, + { + "entropy": 0.556266717761755, + "epoch": 2.7360350492880614, + "grad_norm": 0.5275787115097046, + "learning_rate": 6.892593796297452e-05, + "loss": 0.499769401550293, + "mean_token_accuracy": 0.8451313543319702, + "num_tokens": 2980972.0, + "step": 1250 + }, + { + "entropy": 0.5362468618154526, + "epoch": 2.845564074479737, + "grad_norm": 0.5450661182403564, + "learning_rate": 6.808203529425189e-05, + "loss": 0.4860528945922852, + "mean_token_accuracy": 0.8479894894361496, + "num_tokens": 3108708.0, + "step": 1300 + }, + { + "entropy": 0.5334719524532556, + "epoch": 2.955093099671413, + "grad_norm": 0.4825150966644287, + "learning_rate": 6.719399738649542e-05, + "loss": 0.48385780334472656, + "mean_token_accuracy": 0.8489498183131218, + "num_tokens": 3232991.0, + "step": 1350 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5111288797287714, + "eval_loss": 0.5738435387611389, + "eval_mean_token_accuracy": 0.8259959235077813, + "eval_num_tokens": 3283422.0, + "eval_runtime": 95.6002, + "eval_samples_per_second": 10.481, + "eval_steps_per_second": 1.318, + "step": 1371 + }, + { + "entropy": 0.5024978819519582, + "epoch": 3.063526834611172, + "grad_norm": 0.6228470802307129, + "learning_rate": 6.626311933299292e-05, + "loss": 0.4451956939697266, + "mean_token_accuracy": 0.8579568441468056, + "num_tokens": 3352672.0, + "step": 1400 + }, + { + "entropy": 0.49351966604590414, + "epoch": 3.1730558598028478, + "grad_norm": 0.648960530757904, + "learning_rate": 6.529075870407823e-05, + "loss": 0.4324279022216797, + "mean_token_accuracy": 0.8607994091510772, + "num_tokens": 3472463.0, + "step": 1450 + }, + { + "entropy": 0.48342301592230796, + "epoch": 3.2825848849945234, + "grad_norm": 0.8443573713302612, + "learning_rate": 6.427833356728302e-05, + "loss": 0.4237791442871094, + "mean_token_accuracy": 0.8643713328242302, + "num_tokens": 3593837.0, + "step": 1500 + }, + { + "entropy": 0.4779162485897541, + "epoch": 3.3921139101861995, + "grad_norm": 0.7882111072540283, + "learning_rate": 6.32273204192609e-05, + "loss": 0.42386363983154296, + "mean_token_accuracy": 0.8635856115818024, + "num_tokens": 3716058.0, + "step": 1550 + }, + { + "entropy": 0.48870255261659623, + "epoch": 3.501642935377875, + "grad_norm": 0.7637454867362976, + "learning_rate": 6.213925203250001e-05, + "loss": 0.4301974105834961, + "mean_token_accuracy": 0.861739870607853, + "num_tokens": 3838629.0, + "step": 1600 + }, + { + "entropy": 0.49970791533589365, + "epoch": 3.6111719605695507, + "grad_norm": 0.7269095182418823, + "learning_rate": 6.101571521996419e-05, + "loss": 0.4372034454345703, + "mean_token_accuracy": 0.8592326313257217, + "num_tokens": 3955462.0, + "step": 1650 + }, + { + "entropy": 0.5049038740992546, + "epoch": 3.7207009857612268, + "grad_norm": 0.6023766398429871, + "learning_rate": 5.98583485209228e-05, + "loss": 0.445910758972168, + "mean_token_accuracy": 0.8579568776488304, + "num_tokens": 4075933.0, + "step": 1700 + }, + { + "entropy": 0.4927579787373543, + "epoch": 3.8302300109529024, + "grad_norm": 0.7742004990577698, + "learning_rate": 5.866883981134422e-05, + "loss": 0.4280668640136719, + "mean_token_accuracy": 0.86241753667593, + "num_tokens": 4192915.0, + "step": 1750 + }, + { + "entropy": 0.5011553263664246, + "epoch": 3.9397590361445785, + "grad_norm": 0.5748111605644226, + "learning_rate": 5.7448923842337736e-05, + "loss": 0.43842597961425783, + "mean_token_accuracy": 0.8605782136321067, + "num_tokens": 4310431.0, + "step": 1800 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4624898036321004, + "eval_loss": 0.590175986289978, + "eval_mean_token_accuracy": 0.8287760076068696, + "eval_num_tokens": 4377896.0, + "eval_runtime": 95.3934, + "eval_samples_per_second": 10.504, + "eval_steps_per_second": 1.321, + "step": 1828 + }, + { + "entropy": 0.4502198097079691, + "epoch": 4.048192771084337, + "grad_norm": 0.7170541882514954, + "learning_rate": 5.620037971023403e-05, + "loss": 0.38744712829589845, + "mean_token_accuracy": 0.8735785065877317, + "num_tokens": 4430089.0, + "step": 1850 + }, + { + "entropy": 0.41583337262272835, + "epoch": 4.157721796276014, + "grad_norm": 0.9511433243751526, + "learning_rate": 5.4925028261993515e-05, + "loss": 0.3562023162841797, + "mean_token_accuracy": 0.881559683084488, + "num_tokens": 4554484.0, + "step": 1900 + }, + { + "entropy": 0.4208242034912109, + "epoch": 4.267250821467689, + "grad_norm": 0.8741114139556885, + "learning_rate": 5.3624729439726544e-05, + "loss": 0.3612668991088867, + "mean_token_accuracy": 0.8802913293242455, + "num_tokens": 4675472.0, + "step": 1950 + }, + { + "entropy": 0.4188448017835617, + "epoch": 4.376779846659365, + "grad_norm": 1.1690106391906738, + "learning_rate": 5.23013795681983e-05, + "loss": 0.3627183151245117, + "mean_token_accuracy": 0.8805234292149544, + "num_tokens": 4793479.0, + "step": 2000 + }, + { + "entropy": 0.42332509815692904, + "epoch": 4.48630887185104, + "grad_norm": 0.8150995969772339, + "learning_rate": 5.095690858927403e-05, + "loss": 0.3626524353027344, + "mean_token_accuracy": 0.879372145831585, + "num_tokens": 4911343.0, + "step": 2050 + }, + { + "entropy": 0.42394075110554696, + "epoch": 4.595837897042716, + "grad_norm": 0.8282762169837952, + "learning_rate": 4.959327724733778e-05, + "loss": 0.3573355865478516, + "mean_token_accuracy": 0.8799301481246948, + "num_tokens": 5028364.0, + "step": 2100 + }, + { + "entropy": 0.4259473057091236, + "epoch": 4.705366922234392, + "grad_norm": 0.7610743045806885, + "learning_rate": 4.8212474229789754e-05, + "loss": 0.3665072631835937, + "mean_token_accuracy": 0.879306109547615, + "num_tokens": 5146995.0, + "step": 2150 + }, + { + "entropy": 0.42407046899199485, + "epoch": 4.814895947426068, + "grad_norm": 0.6995375156402588, + "learning_rate": 4.681651326679193e-05, + "loss": 0.3689637756347656, + "mean_token_accuracy": 0.878270491361618, + "num_tokens": 5263483.0, + "step": 2200 + }, + { + "entropy": 0.41719858527183534, + "epoch": 4.924424972617744, + "grad_norm": 0.8921851515769958, + "learning_rate": 4.5407430194492145e-05, + "loss": 0.36366527557373046, + "mean_token_accuracy": 0.8810832899808884, + "num_tokens": 5387955.0, + "step": 2250 + }, + { + "epoch": 5.0, + "eval_entropy": 0.40321539318750776, + "eval_loss": 0.6344199776649475, + "eval_mean_token_accuracy": 0.8290003571245406, + "eval_num_tokens": 5472370.0, + "eval_runtime": 95.4906, + "eval_samples_per_second": 10.493, + "eval_steps_per_second": 1.32, + "step": 2285 + }, + { + "entropy": 0.39761772337887025, + "epoch": 5.032858707557502, + "grad_norm": 0.8363515734672546, + "learning_rate": 4.3987279986009235e-05, + "loss": 0.3363536834716797, + "mean_token_accuracy": 0.8891537577816935, + "num_tokens": 5508058.0, + "step": 2300 + }, + { + "entropy": 0.3406807939708233, + "epoch": 5.142387732749179, + "grad_norm": 0.9140804409980774, + "learning_rate": 4.2558133754509274e-05, + "loss": 0.2747584533691406, + "mean_token_accuracy": 0.9082898917794228, + "num_tokens": 5625635.0, + "step": 2350 + }, + { + "entropy": 0.34594309888780117, + "epoch": 5.2519167579408546, + "grad_norm": 0.9676663875579834, + "learning_rate": 4.112207573274355e-05, + "loss": 0.2821139907836914, + "mean_token_accuracy": 0.9047485241293907, + "num_tokens": 5741941.0, + "step": 2400 + }, + { + "entropy": 0.3354563079029322, + "epoch": 5.36144578313253, + "grad_norm": 1.1732304096221924, + "learning_rate": 3.968120023345335e-05, + "loss": 0.2757284355163574, + "mean_token_accuracy": 0.905784958600998, + "num_tokens": 5864741.0, + "step": 2450 + }, + { + "entropy": 0.3486690762639046, + "epoch": 5.470974808324206, + "grad_norm": 0.9858622550964355, + "learning_rate": 3.823760859507414e-05, + "loss": 0.28488592147827146, + "mean_token_accuracy": 0.9029894617199897, + "num_tokens": 5984809.0, + "step": 2500 + }, + { + "entropy": 0.35116296328604224, + "epoch": 5.580503833515881, + "grad_norm": 0.8295992016792297, + "learning_rate": 3.679340611719382e-05, + "loss": 0.28818355560302733, + "mean_token_accuracy": 0.9026859793066978, + "num_tokens": 6101316.0, + "step": 2550 + }, + { + "entropy": 0.34762901581823824, + "epoch": 5.690032858707557, + "grad_norm": 1.2317472696304321, + "learning_rate": 3.5350698990234046e-05, + "loss": 0.2834972381591797, + "mean_token_accuracy": 0.9035492998361587, + "num_tokens": 6222300.0, + "step": 2600 + }, + { + "entropy": 0.3509623434394598, + "epoch": 5.7995618838992335, + "grad_norm": 1.1071515083312988, + "learning_rate": 3.391159122383239e-05, + "loss": 0.28417932510375976, + "mean_token_accuracy": 0.9017772257328034, + "num_tokens": 6339706.0, + "step": 2650 + }, + { + "entropy": 0.3327887299656868, + "epoch": 5.909090909090909, + "grad_norm": 0.9410860538482666, + "learning_rate": 3.247818157840487e-05, + "loss": 0.27511814117431643, + "mean_token_accuracy": 0.9069757598638535, + "num_tokens": 6466571.0, + "step": 2700 + }, + { + "epoch": 6.0, + "eval_entropy": 0.37359742701999726, + "eval_loss": 0.6801024079322815, + "eval_mean_token_accuracy": 0.8251904424220796, + "eval_num_tokens": 6566844.0, + "eval_runtime": 95.5582, + "eval_samples_per_second": 10.486, + "eval_steps_per_second": 1.319, + "step": 2742 + }, + { + "entropy": 0.33517336732510367, + "epoch": 6.017524644030668, + "grad_norm": 1.1120449304580688, + "learning_rate": 3.105256050436392e-05, + "loss": 0.2726051139831543, + "mean_token_accuracy": 0.9074887001153195, + "num_tokens": 6588372.0, + "step": 2750 + }, + { + "entropy": 0.2580332762002945, + "epoch": 6.127053669222344, + "grad_norm": 1.1544626951217651, + "learning_rate": 2.9636807093455337e-05, + "loss": 0.1894158172607422, + "mean_token_accuracy": 0.934169539809227, + "num_tokens": 6712562.0, + "step": 2800 + }, + { + "entropy": 0.2648825005441904, + "epoch": 6.23658269441402, + "grad_norm": 1.2022897005081177, + "learning_rate": 2.823298604666056e-05, + "loss": 0.19440870285034179, + "mean_token_accuracy": 0.9334753274917602, + "num_tokens": 6832843.0, + "step": 2850 + }, + { + "entropy": 0.27026796594262126, + "epoch": 6.3461117196056955, + "grad_norm": 1.0150978565216064, + "learning_rate": 2.6843144663086045e-05, + "loss": 0.19671850204467772, + "mean_token_accuracy": 0.9316870296001434, + "num_tokens": 6952871.0, + "step": 2900 + }, + { + "entropy": 0.2634010723978281, + "epoch": 6.455640744797371, + "grad_norm": 1.430558681488037, + "learning_rate": 2.546930985423105e-05, + "loss": 0.19457483291625977, + "mean_token_accuracy": 0.9327938884496689, + "num_tokens": 7072067.0, + "step": 2950 + }, + { + "entropy": 0.2702385004609823, + "epoch": 6.565169769989047, + "grad_norm": 1.0556726455688477, + "learning_rate": 2.4113485187988342e-05, + "loss": 0.19928800582885742, + "mean_token_accuracy": 0.9298818710446358, + "num_tokens": 7187280.0, + "step": 3000 + }, + { + "entropy": 0.2702864905446768, + "epoch": 6.674698795180722, + "grad_norm": 1.1694364547729492, + "learning_rate": 2.2777647966688595e-05, + "loss": 0.20284730911254883, + "mean_token_accuracy": 0.9301017987728118, + "num_tokens": 7305186.0, + "step": 3050 + }, + { + "entropy": 0.2645207424461842, + "epoch": 6.784227820372399, + "grad_norm": 1.2039296627044678, + "learning_rate": 2.146374634344989e-05, + "loss": 0.1961233139038086, + "mean_token_accuracy": 0.9320311924815178, + "num_tokens": 7427803.0, + "step": 3100 + }, + { + "entropy": 0.2703990802913904, + "epoch": 6.8937568455640745, + "grad_norm": 1.122693657875061, + "learning_rate": 2.01736964810376e-05, + "loss": 0.19937246322631835, + "mean_token_accuracy": 0.9296649679541588, + "num_tokens": 7543735.0, + "step": 3150 + }, + { + "epoch": 7.0, + "eval_entropy": 0.31011983941471766, + "eval_loss": 0.7977674007415771, + "eval_mean_token_accuracy": 0.8200774944963909, + "eval_num_tokens": 7661318.0, + "eval_runtime": 95.5905, + "eval_samples_per_second": 10.482, + "eval_steps_per_second": 1.318, + "step": 3199 + }, + { + "entropy": 0.27026369738759415, + "epoch": 7.002190580503833, + "grad_norm": 0.8098449110984802, + "learning_rate": 1.8909379757378543e-05, + "loss": 0.19773469924926756, + "mean_token_accuracy": 0.9312729013688636, + "num_tokens": 7663078.0, + "step": 3200 + }, + { + "entropy": 0.21044189006090164, + "epoch": 7.11171960569551, + "grad_norm": 1.2920125722885132, + "learning_rate": 1.7672640021804126e-05, + "loss": 0.13121842384338378, + "mean_token_accuracy": 0.954786310493946, + "num_tokens": 7782363.0, + "step": 3250 + }, + { + "entropy": 0.2033486431464553, + "epoch": 7.221248630887185, + "grad_norm": 1.1921308040618896, + "learning_rate": 1.6465280906024633e-05, + "loss": 0.13039697647094728, + "mean_token_accuracy": 0.9554795321822166, + "num_tokens": 7899912.0, + "step": 3300 + }, + { + "entropy": 0.2082062489911914, + "epoch": 7.330777656078861, + "grad_norm": 1.1978024244308472, + "learning_rate": 1.5289063193755705e-05, + "loss": 0.13286386489868163, + "mean_token_accuracy": 0.9534762743115425, + "num_tokens": 8019370.0, + "step": 3350 + }, + { + "entropy": 0.2054990328848362, + "epoch": 7.4403066812705365, + "grad_norm": 1.1366864442825317, + "learning_rate": 1.4145702252833402e-05, + "loss": 0.1314125633239746, + "mean_token_accuracy": 0.9538632571697235, + "num_tokens": 8142751.0, + "step": 3400 + }, + { + "entropy": 0.20359413929283618, + "epoch": 7.549835706462212, + "grad_norm": 1.2026054859161377, + "learning_rate": 1.3036865533562794e-05, + "loss": 0.1330260467529297, + "mean_token_accuracy": 0.9539996239542962, + "num_tokens": 8260948.0, + "step": 3450 + }, + { + "entropy": 0.21044832307845354, + "epoch": 7.659364731653889, + "grad_norm": 1.3045598268508911, + "learning_rate": 1.1964170136948073e-05, + "loss": 0.13348891258239745, + "mean_token_accuracy": 0.9526105988025665, + "num_tokens": 8378334.0, + "step": 3500 + }, + { + "entropy": 0.20063834685832263, + "epoch": 7.768893756845564, + "grad_norm": 1.001644253730774, + "learning_rate": 1.0929180456351225e-05, + "loss": 0.12778966903686523, + "mean_token_accuracy": 0.955841820538044, + "num_tokens": 8503618.0, + "step": 3550 + }, + { + "entropy": 0.20864798523485661, + "epoch": 7.87842278203724, + "grad_norm": 1.0793193578720093, + "learning_rate": 9.933405896017934e-06, + "loss": 0.13261634826660157, + "mean_token_accuracy": 0.9527471828460693, + "num_tokens": 8621388.0, + "step": 3600 + }, + { + "entropy": 0.2030112524703145, + "epoch": 7.9879518072289155, + "grad_norm": 0.9351119995117188, + "learning_rate": 8.978298669798568e-06, + "loss": 0.13032340049743651, + "mean_token_accuracy": 0.953511869609356, + "num_tokens": 8743662.0, + "step": 3650 + }, + { + "epoch": 8.0, + "eval_entropy": 0.27168176760749213, + "eval_loss": 0.9082825779914856, + "eval_mean_token_accuracy": 0.8165905882441808, + "eval_num_tokens": 8755792.0, + "eval_runtime": 95.6338, + "eval_samples_per_second": 10.477, + "eval_steps_per_second": 1.318, + "step": 3656 + }, + { + "entropy": 0.17589674077250742, + "epoch": 8.096385542168674, + "grad_norm": 0.8960739374160767, + "learning_rate": 8.065251683274092e-06, + "loss": 0.09905588150024414, + "mean_token_accuracy": 0.9655855659282568, + "num_tokens": 8864043.0, + "step": 3700 + }, + { + "entropy": 0.1714504823088646, + "epoch": 8.20591456736035, + "grad_norm": 0.9031808376312256, + "learning_rate": 7.195596502375899e-06, + "loss": 0.09377479553222656, + "mean_token_accuracy": 0.9666973549127579, + "num_tokens": 8984809.0, + "step": 3750 + }, + { + "entropy": 0.16023088485002518, + "epoch": 8.315443592552027, + "grad_norm": 1.0721874237060547, + "learning_rate": 6.370601411461736e-06, + "loss": 0.09197239875793457, + "mean_token_accuracy": 0.9688060078024864, + "num_tokens": 9108507.0, + "step": 3800 + }, + { + "entropy": 0.17518360134214161, + "epoch": 8.424972617743702, + "grad_norm": 0.7766899466514587, + "learning_rate": 5.5914695636801535e-06, + "loss": 0.09632173538208008, + "mean_token_accuracy": 0.964902586042881, + "num_tokens": 9226857.0, + "step": 3850 + }, + { + "entropy": 0.16780287627130747, + "epoch": 8.534501642935378, + "grad_norm": 0.8962944149971008, + "learning_rate": 4.859337226320612e-06, + "loss": 0.09255614280700683, + "mean_token_accuracy": 0.9672093752026558, + "num_tokens": 9349892.0, + "step": 3900 + }, + { + "entropy": 0.17159098640084267, + "epoch": 8.644030668127053, + "grad_norm": 0.965328574180603, + "learning_rate": 4.175272123708463e-06, + "loss": 0.0957132911682129, + "mean_token_accuracy": 0.9660508191585541, + "num_tokens": 9464026.0, + "step": 3950 + }, + { + "entropy": 0.16975272938609123, + "epoch": 8.75355969331873, + "grad_norm": 0.7491775751113892, + "learning_rate": 3.540271880061184e-06, + "loss": 0.09802600860595703, + "mean_token_accuracy": 0.9656609424948692, + "num_tokens": 9577583.0, + "step": 4000 + }, + { + "entropy": 0.16477238953113557, + "epoch": 8.863088718510404, + "grad_norm": 1.2014434337615967, + "learning_rate": 2.9552625645771213e-06, + "loss": 0.09635594367980957, + "mean_token_accuracy": 0.966949755847454, + "num_tokens": 9694696.0, + "step": 4050 + }, + { + "entropy": 0.16196822334080935, + "epoch": 8.97261774370208, + "grad_norm": 0.9184058308601379, + "learning_rate": 2.421097340878212e-06, + "loss": 0.09237998008728027, + "mean_token_accuracy": 0.9681418687105179, + "num_tokens": 9819102.0, + "step": 4100 + }, + { + "epoch": 9.0, + "eval_entropy": 0.23909894093161538, + "eval_loss": 1.0375252962112427, + "eval_mean_token_accuracy": 0.8140787217352126, + "eval_num_tokens": 9850266.0, + "eval_runtime": 95.5931, + "eval_samples_per_second": 10.482, + "eval_steps_per_second": 1.318, + "step": 4113 + } + ], + "logging_steps": 50, + "max_steps": 4570, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6331801930428723e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/README.md b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/adapter_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a72a4c543bf8671cb7019360179c8cabb6e5d6c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.021223473447411947, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "gate_proj", + "v_proj", + "up_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/chat_template.jinja b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/tokenizer_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/trainer_state.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..15ebd6fef315c4bdbe3355ab1b7dbe52c71082b3 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-457/trainer_state.json @@ -0,0 +1,135 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 457, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.002540482878685, + "epoch": 0.10952902519167579, + "grad_norm": 0.5552461743354797, + "learning_rate": 8.109930924389919e-06, + "loss": 1.9067156982421876, + "mean_token_accuracy": 0.6026969534158707, + "num_tokens": 119060.0, + "step": 50 + }, + { + "entropy": 1.3068562260270118, + "epoch": 0.21905805038335158, + "grad_norm": 1.0465189218521118, + "learning_rate": 1.6385370643155144e-05, + "loss": 1.1655167388916015, + "mean_token_accuracy": 0.7124340075254441, + "num_tokens": 242713.0, + "step": 100 + }, + { + "entropy": 0.8696084088087082, + "epoch": 0.32858707557502737, + "grad_norm": 0.6686795949935913, + "learning_rate": 2.4660810361920367e-05, + "loss": 0.8031976318359375, + "mean_token_accuracy": 0.7754033240675926, + "num_tokens": 360160.0, + "step": 150 + }, + { + "entropy": 0.7853927117586136, + "epoch": 0.43811610076670315, + "grad_norm": 0.9354436993598938, + "learning_rate": 3.293625008068559e-05, + "loss": 0.7226885986328125, + "mean_token_accuracy": 0.791197614967823, + "num_tokens": 485403.0, + "step": 200 + }, + { + "entropy": 0.7382483741641045, + "epoch": 0.547645125958379, + "grad_norm": 0.7535398006439209, + "learning_rate": 4.121168979945081e-05, + "loss": 0.6755471801757813, + "mean_token_accuracy": 0.8054334259033203, + "num_tokens": 601578.0, + "step": 250 + }, + { + "entropy": 0.7218390592932701, + "epoch": 0.6571741511500547, + "grad_norm": 0.8160243034362793, + "learning_rate": 4.948712951821604e-05, + "loss": 0.6614183044433594, + "mean_token_accuracy": 0.8060630604624748, + "num_tokens": 713917.0, + "step": 300 + }, + { + "entropy": 0.7012957927584648, + "epoch": 0.7667031763417306, + "grad_norm": 0.7985761165618896, + "learning_rate": 5.776256923698126e-05, + "loss": 0.6415711975097657, + "mean_token_accuracy": 0.8100408402085304, + "num_tokens": 832995.0, + "step": 350 + }, + { + "entropy": 0.6763210469484329, + "epoch": 0.8762322015334063, + "grad_norm": 0.6829700469970703, + "learning_rate": 6.603800895574648e-05, + "loss": 0.6193617248535156, + "mean_token_accuracy": 0.8140933158993721, + "num_tokens": 959304.0, + "step": 400 + }, + { + "entropy": 0.6874829810857773, + "epoch": 0.9857612267250822, + "grad_norm": 0.6983632445335388, + "learning_rate": 7.43134486745117e-05, + "loss": 0.6262834167480469, + "mean_token_accuracy": 0.8163631609082223, + "num_tokens": 1078880.0, + "step": 450 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6085402492492918, + "eval_loss": 0.6388216018676758, + "eval_mean_token_accuracy": 0.8123693073552752, + "eval_num_tokens": 1094474.0, + "eval_runtime": 95.9846, + "eval_samples_per_second": 10.439, + "eval_steps_per_second": 1.313, + "step": 457 + } + ], + "logging_steps": 50, + "max_steps": 4570, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.8043367722576896e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/README.md b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/adapter_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a72a4c543bf8671cb7019360179c8cabb6e5d6c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.021223473447411947, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "gate_proj", + "v_proj", + "up_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/chat_template.jinja b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/tokenizer_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/trainer_state.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..63e1eb4f85e2155d15bc998549e9ad42370eba32 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-4570/trainer_state.json @@ -0,0 +1,1054 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4570, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.002540482878685, + "epoch": 0.10952902519167579, + "grad_norm": 0.5552461743354797, + "learning_rate": 8.109930924389919e-06, + "loss": 1.9067156982421876, + "mean_token_accuracy": 0.6026969534158707, + "num_tokens": 119060.0, + "step": 50 + }, + { + "entropy": 1.3068562260270118, + "epoch": 0.21905805038335158, + "grad_norm": 1.0465189218521118, + "learning_rate": 1.6385370643155144e-05, + "loss": 1.1655167388916015, + "mean_token_accuracy": 0.7124340075254441, + "num_tokens": 242713.0, + "step": 100 + }, + { + "entropy": 0.8696084088087082, + "epoch": 0.32858707557502737, + "grad_norm": 0.6686795949935913, + "learning_rate": 2.4660810361920367e-05, + "loss": 0.8031976318359375, + "mean_token_accuracy": 0.7754033240675926, + "num_tokens": 360160.0, + "step": 150 + }, + { + "entropy": 0.7853927117586136, + "epoch": 0.43811610076670315, + "grad_norm": 0.9354436993598938, + "learning_rate": 3.293625008068559e-05, + "loss": 0.7226885986328125, + "mean_token_accuracy": 0.791197614967823, + "num_tokens": 485403.0, + "step": 200 + }, + { + "entropy": 0.7382483741641045, + "epoch": 0.547645125958379, + "grad_norm": 0.7535398006439209, + "learning_rate": 4.121168979945081e-05, + "loss": 0.6755471801757813, + "mean_token_accuracy": 0.8054334259033203, + "num_tokens": 601578.0, + "step": 250 + }, + { + "entropy": 0.7218390592932701, + "epoch": 0.6571741511500547, + "grad_norm": 0.8160243034362793, + "learning_rate": 4.948712951821604e-05, + "loss": 0.6614183044433594, + "mean_token_accuracy": 0.8060630604624748, + "num_tokens": 713917.0, + "step": 300 + }, + { + "entropy": 0.7012957927584648, + "epoch": 0.7667031763417306, + "grad_norm": 0.7985761165618896, + "learning_rate": 5.776256923698126e-05, + "loss": 0.6415711975097657, + "mean_token_accuracy": 0.8100408402085304, + "num_tokens": 832995.0, + "step": 350 + }, + { + "entropy": 0.6763210469484329, + "epoch": 0.8762322015334063, + "grad_norm": 0.6829700469970703, + "learning_rate": 6.603800895574648e-05, + "loss": 0.6193617248535156, + "mean_token_accuracy": 0.8140933158993721, + "num_tokens": 959304.0, + "step": 400 + }, + { + "entropy": 0.6874829810857773, + "epoch": 0.9857612267250822, + "grad_norm": 0.6983632445335388, + "learning_rate": 7.43134486745117e-05, + "loss": 0.6262834167480469, + "mean_token_accuracy": 0.8163631609082223, + "num_tokens": 1078880.0, + "step": 450 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6085402492492918, + "eval_loss": 0.6388216018676758, + "eval_mean_token_accuracy": 0.8123693073552752, + "eval_num_tokens": 1094474.0, + "eval_runtime": 95.9846, + "eval_samples_per_second": 10.439, + "eval_steps_per_second": 1.313, + "step": 457 + }, + { + "entropy": 0.6471366012337232, + "epoch": 1.0941949616648412, + "grad_norm": 0.7878272533416748, + "learning_rate": 7.561806001041411e-05, + "loss": 0.589820671081543, + "mean_token_accuracy": 0.8218218798589225, + "num_tokens": 1190065.0, + "step": 500 + }, + { + "entropy": 0.6404865515232087, + "epoch": 1.203723986856517, + "grad_norm": 0.7976964712142944, + "learning_rate": 7.554418144822605e-05, + "loss": 0.5836894607543945, + "mean_token_accuracy": 0.8222010856866837, + "num_tokens": 1313079.0, + "step": 550 + }, + { + "entropy": 0.6355892798304558, + "epoch": 1.3132530120481927, + "grad_norm": 0.7428044080734253, + "learning_rate": 7.541528503116934e-05, + "loss": 0.577253189086914, + "mean_token_accuracy": 0.8260676205158234, + "num_tokens": 1428305.0, + "step": 600 + }, + { + "entropy": 0.6170084626972675, + "epoch": 1.4227820372398685, + "grad_norm": 0.6792078614234924, + "learning_rate": 7.523155873870194e-05, + "loss": 0.561871337890625, + "mean_token_accuracy": 0.8307154527306557, + "num_tokens": 1551971.0, + "step": 650 + }, + { + "entropy": 0.6222476975619793, + "epoch": 1.5323110624315444, + "grad_norm": 0.6532447934150696, + "learning_rate": 7.499327051286336e-05, + "loss": 0.5612493515014648, + "mean_token_accuracy": 0.8296262130141259, + "num_tokens": 1673401.0, + "step": 700 + }, + { + "entropy": 0.6251883202791214, + "epoch": 1.6418400876232202, + "grad_norm": 0.7107782959938049, + "learning_rate": 7.47007678675144e-05, + "loss": 0.565279769897461, + "mean_token_accuracy": 0.8299687370657921, + "num_tokens": 1788813.0, + "step": 750 + }, + { + "entropy": 0.6177104935050011, + "epoch": 1.751369112814896, + "grad_norm": 0.7850057482719421, + "learning_rate": 7.435447738153122e-05, + "loss": 0.5564990234375, + "mean_token_accuracy": 0.8291967037320137, + "num_tokens": 1911556.0, + "step": 800 + }, + { + "entropy": 0.6029443763196468, + "epoch": 1.8608981380065717, + "grad_norm": 0.7116957306861877, + "learning_rate": 7.395490407669285e-05, + "loss": 0.5501844406127929, + "mean_token_accuracy": 0.8335196697711944, + "num_tokens": 2029129.0, + "step": 850 + }, + { + "entropy": 0.6042105440795421, + "epoch": 1.9704271631982475, + "grad_norm": 0.6712159514427185, + "learning_rate": 7.350263068116955e-05, + "loss": 0.5517086029052735, + "mean_token_accuracy": 0.8321291375160217, + "num_tokens": 2155392.0, + "step": 900 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5401396741942753, + "eval_loss": 0.5797445774078369, + "eval_mean_token_accuracy": 0.8243241475688087, + "eval_num_tokens": 2188948.0, + "eval_runtime": 95.5407, + "eval_samples_per_second": 10.488, + "eval_steps_per_second": 1.319, + "step": 914 + }, + { + "entropy": 0.5675096463675451, + "epoch": 2.078860898138007, + "grad_norm": 0.5713562369346619, + "learning_rate": 7.299831677968588e-05, + "loss": 0.5120392227172852, + "mean_token_accuracy": 0.8414448993374603, + "num_tokens": 2277503.0, + "step": 950 + }, + { + "entropy": 0.5500032117962838, + "epoch": 2.1883899233296824, + "grad_norm": 0.5951120257377625, + "learning_rate": 7.244269785159817e-05, + "loss": 0.49384498596191406, + "mean_token_accuracy": 0.8470860269665718, + "num_tokens": 2392968.0, + "step": 1000 + }, + { + "entropy": 0.5683873899281024, + "epoch": 2.297918948521358, + "grad_norm": 0.6816521286964417, + "learning_rate": 7.183658419828891e-05, + "loss": 0.5088459014892578, + "mean_token_accuracy": 0.8413213565945625, + "num_tokens": 2507108.0, + "step": 1050 + }, + { + "entropy": 0.5481136417388917, + "epoch": 2.407447973713034, + "grad_norm": 0.6417970657348633, + "learning_rate": 7.118085976144257e-05, + "loss": 0.49456378936767575, + "mean_token_accuracy": 0.8468478980660439, + "num_tokens": 2633824.0, + "step": 1100 + }, + { + "entropy": 0.5413966289162636, + "epoch": 2.5169769989047097, + "grad_norm": 0.631996214389801, + "learning_rate": 7.047648083392619e-05, + "loss": 0.49154373168945314, + "mean_token_accuracy": 0.8461908429861069, + "num_tokens": 2753131.0, + "step": 1150 + }, + { + "entropy": 0.5583206915855408, + "epoch": 2.6265060240963853, + "grad_norm": 0.7409902215003967, + "learning_rate": 6.972447466515462e-05, + "loss": 0.4927285385131836, + "mean_token_accuracy": 0.8451325806975365, + "num_tokens": 2865112.0, + "step": 1200 + }, + { + "entropy": 0.556266717761755, + "epoch": 2.7360350492880614, + "grad_norm": 0.5275787115097046, + "learning_rate": 6.892593796297452e-05, + "loss": 0.499769401550293, + "mean_token_accuracy": 0.8451313543319702, + "num_tokens": 2980972.0, + "step": 1250 + }, + { + "entropy": 0.5362468618154526, + "epoch": 2.845564074479737, + "grad_norm": 0.5450661182403564, + "learning_rate": 6.808203529425189e-05, + "loss": 0.4860528945922852, + "mean_token_accuracy": 0.8479894894361496, + "num_tokens": 3108708.0, + "step": 1300 + }, + { + "entropy": 0.5334719524532556, + "epoch": 2.955093099671413, + "grad_norm": 0.4825150966644287, + "learning_rate": 6.719399738649542e-05, + "loss": 0.48385780334472656, + "mean_token_accuracy": 0.8489498183131218, + "num_tokens": 3232991.0, + "step": 1350 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5111288797287714, + "eval_loss": 0.5738435387611389, + "eval_mean_token_accuracy": 0.8259959235077813, + "eval_num_tokens": 3283422.0, + "eval_runtime": 95.6002, + "eval_samples_per_second": 10.481, + "eval_steps_per_second": 1.318, + "step": 1371 + }, + { + "entropy": 0.5024978819519582, + "epoch": 3.063526834611172, + "grad_norm": 0.6228470802307129, + "learning_rate": 6.626311933299292e-05, + "loss": 0.4451956939697266, + "mean_token_accuracy": 0.8579568441468056, + "num_tokens": 3352672.0, + "step": 1400 + }, + { + "entropy": 0.49351966604590414, + "epoch": 3.1730558598028478, + "grad_norm": 0.648960530757904, + "learning_rate": 6.529075870407823e-05, + "loss": 0.4324279022216797, + "mean_token_accuracy": 0.8607994091510772, + "num_tokens": 3472463.0, + "step": 1450 + }, + { + "entropy": 0.48342301592230796, + "epoch": 3.2825848849945234, + "grad_norm": 0.8443573713302612, + "learning_rate": 6.427833356728302e-05, + "loss": 0.4237791442871094, + "mean_token_accuracy": 0.8643713328242302, + "num_tokens": 3593837.0, + "step": 1500 + }, + { + "entropy": 0.4779162485897541, + "epoch": 3.3921139101861995, + "grad_norm": 0.7882111072540283, + "learning_rate": 6.32273204192609e-05, + "loss": 0.42386363983154296, + "mean_token_accuracy": 0.8635856115818024, + "num_tokens": 3716058.0, + "step": 1550 + }, + { + "entropy": 0.48870255261659623, + "epoch": 3.501642935377875, + "grad_norm": 0.7637454867362976, + "learning_rate": 6.213925203250001e-05, + "loss": 0.4301974105834961, + "mean_token_accuracy": 0.861739870607853, + "num_tokens": 3838629.0, + "step": 1600 + }, + { + "entropy": 0.49970791533589365, + "epoch": 3.6111719605695507, + "grad_norm": 0.7269095182418823, + "learning_rate": 6.101571521996419e-05, + "loss": 0.4372034454345703, + "mean_token_accuracy": 0.8592326313257217, + "num_tokens": 3955462.0, + "step": 1650 + }, + { + "entropy": 0.5049038740992546, + "epoch": 3.7207009857612268, + "grad_norm": 0.6023766398429871, + "learning_rate": 5.98583485209228e-05, + "loss": 0.445910758972168, + "mean_token_accuracy": 0.8579568776488304, + "num_tokens": 4075933.0, + "step": 1700 + }, + { + "entropy": 0.4927579787373543, + "epoch": 3.8302300109529024, + "grad_norm": 0.7742004990577698, + "learning_rate": 5.866883981134422e-05, + "loss": 0.4280668640136719, + "mean_token_accuracy": 0.86241753667593, + "num_tokens": 4192915.0, + "step": 1750 + }, + { + "entropy": 0.5011553263664246, + "epoch": 3.9397590361445785, + "grad_norm": 0.5748111605644226, + "learning_rate": 5.7448923842337736e-05, + "loss": 0.43842597961425783, + "mean_token_accuracy": 0.8605782136321067, + "num_tokens": 4310431.0, + "step": 1800 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4624898036321004, + "eval_loss": 0.590175986289978, + "eval_mean_token_accuracy": 0.8287760076068696, + "eval_num_tokens": 4377896.0, + "eval_runtime": 95.3934, + "eval_samples_per_second": 10.504, + "eval_steps_per_second": 1.321, + "step": 1828 + }, + { + "entropy": 0.4502198097079691, + "epoch": 4.048192771084337, + "grad_norm": 0.7170541882514954, + "learning_rate": 5.620037971023403e-05, + "loss": 0.38744712829589845, + "mean_token_accuracy": 0.8735785065877317, + "num_tokens": 4430089.0, + "step": 1850 + }, + { + "entropy": 0.41583337262272835, + "epoch": 4.157721796276014, + "grad_norm": 0.9511433243751526, + "learning_rate": 5.4925028261993515e-05, + "loss": 0.3562023162841797, + "mean_token_accuracy": 0.881559683084488, + "num_tokens": 4554484.0, + "step": 1900 + }, + { + "entropy": 0.4208242034912109, + "epoch": 4.267250821467689, + "grad_norm": 0.8741114139556885, + "learning_rate": 5.3624729439726544e-05, + "loss": 0.3612668991088867, + "mean_token_accuracy": 0.8802913293242455, + "num_tokens": 4675472.0, + "step": 1950 + }, + { + "entropy": 0.4188448017835617, + "epoch": 4.376779846659365, + "grad_norm": 1.1690106391906738, + "learning_rate": 5.23013795681983e-05, + "loss": 0.3627183151245117, + "mean_token_accuracy": 0.8805234292149544, + "num_tokens": 4793479.0, + "step": 2000 + }, + { + "entropy": 0.42332509815692904, + "epoch": 4.48630887185104, + "grad_norm": 0.8150995969772339, + "learning_rate": 5.095690858927403e-05, + "loss": 0.3626524353027344, + "mean_token_accuracy": 0.879372145831585, + "num_tokens": 4911343.0, + "step": 2050 + }, + { + "entropy": 0.42394075110554696, + "epoch": 4.595837897042716, + "grad_norm": 0.8282762169837952, + "learning_rate": 4.959327724733778e-05, + "loss": 0.3573355865478516, + "mean_token_accuracy": 0.8799301481246948, + "num_tokens": 5028364.0, + "step": 2100 + }, + { + "entropy": 0.4259473057091236, + "epoch": 4.705366922234392, + "grad_norm": 0.7610743045806885, + "learning_rate": 4.8212474229789754e-05, + "loss": 0.3665072631835937, + "mean_token_accuracy": 0.879306109547615, + "num_tokens": 5146995.0, + "step": 2150 + }, + { + "entropy": 0.42407046899199485, + "epoch": 4.814895947426068, + "grad_norm": 0.6995375156402588, + "learning_rate": 4.681651326679193e-05, + "loss": 0.3689637756347656, + "mean_token_accuracy": 0.878270491361618, + "num_tokens": 5263483.0, + "step": 2200 + }, + { + "entropy": 0.41719858527183534, + "epoch": 4.924424972617744, + "grad_norm": 0.8921851515769958, + "learning_rate": 4.5407430194492145e-05, + "loss": 0.36366527557373046, + "mean_token_accuracy": 0.8810832899808884, + "num_tokens": 5387955.0, + "step": 2250 + }, + { + "epoch": 5.0, + "eval_entropy": 0.40321539318750776, + "eval_loss": 0.6344199776649475, + "eval_mean_token_accuracy": 0.8290003571245406, + "eval_num_tokens": 5472370.0, + "eval_runtime": 95.4906, + "eval_samples_per_second": 10.493, + "eval_steps_per_second": 1.32, + "step": 2285 + }, + { + "entropy": 0.39761772337887025, + "epoch": 5.032858707557502, + "grad_norm": 0.8363515734672546, + "learning_rate": 4.3987279986009235e-05, + "loss": 0.3363536834716797, + "mean_token_accuracy": 0.8891537577816935, + "num_tokens": 5508058.0, + "step": 2300 + }, + { + "entropy": 0.3406807939708233, + "epoch": 5.142387732749179, + "grad_norm": 0.9140804409980774, + "learning_rate": 4.2558133754509274e-05, + "loss": 0.2747584533691406, + "mean_token_accuracy": 0.9082898917794228, + "num_tokens": 5625635.0, + "step": 2350 + }, + { + "entropy": 0.34594309888780117, + "epoch": 5.2519167579408546, + "grad_norm": 0.9676663875579834, + "learning_rate": 4.112207573274355e-05, + "loss": 0.2821139907836914, + "mean_token_accuracy": 0.9047485241293907, + "num_tokens": 5741941.0, + "step": 2400 + }, + { + "entropy": 0.3354563079029322, + "epoch": 5.36144578313253, + "grad_norm": 1.1732304096221924, + "learning_rate": 3.968120023345335e-05, + "loss": 0.2757284355163574, + "mean_token_accuracy": 0.905784958600998, + "num_tokens": 5864741.0, + "step": 2450 + }, + { + "entropy": 0.3486690762639046, + "epoch": 5.470974808324206, + "grad_norm": 0.9858622550964355, + "learning_rate": 3.823760859507414e-05, + "loss": 0.28488592147827146, + "mean_token_accuracy": 0.9029894617199897, + "num_tokens": 5984809.0, + "step": 2500 + }, + { + "entropy": 0.35116296328604224, + "epoch": 5.580503833515881, + "grad_norm": 0.8295992016792297, + "learning_rate": 3.679340611719382e-05, + "loss": 0.28818355560302733, + "mean_token_accuracy": 0.9026859793066978, + "num_tokens": 6101316.0, + "step": 2550 + }, + { + "entropy": 0.34762901581823824, + "epoch": 5.690032858707557, + "grad_norm": 1.2317472696304321, + "learning_rate": 3.5350698990234046e-05, + "loss": 0.2834972381591797, + "mean_token_accuracy": 0.9035492998361587, + "num_tokens": 6222300.0, + "step": 2600 + }, + { + "entropy": 0.3509623434394598, + "epoch": 5.7995618838992335, + "grad_norm": 1.1071515083312988, + "learning_rate": 3.391159122383239e-05, + "loss": 0.28417932510375976, + "mean_token_accuracy": 0.9017772257328034, + "num_tokens": 6339706.0, + "step": 2650 + }, + { + "entropy": 0.3327887299656868, + "epoch": 5.909090909090909, + "grad_norm": 0.9410860538482666, + "learning_rate": 3.247818157840487e-05, + "loss": 0.27511814117431643, + "mean_token_accuracy": 0.9069757598638535, + "num_tokens": 6466571.0, + "step": 2700 + }, + { + "epoch": 6.0, + "eval_entropy": 0.37359742701999726, + "eval_loss": 0.6801024079322815, + "eval_mean_token_accuracy": 0.8251904424220796, + "eval_num_tokens": 6566844.0, + "eval_runtime": 95.5582, + "eval_samples_per_second": 10.486, + "eval_steps_per_second": 1.319, + "step": 2742 + }, + { + "entropy": 0.33517336732510367, + "epoch": 6.017524644030668, + "grad_norm": 1.1120449304580688, + "learning_rate": 3.105256050436392e-05, + "loss": 0.2726051139831543, + "mean_token_accuracy": 0.9074887001153195, + "num_tokens": 6588372.0, + "step": 2750 + }, + { + "entropy": 0.2580332762002945, + "epoch": 6.127053669222344, + "grad_norm": 1.1544626951217651, + "learning_rate": 2.9636807093455337e-05, + "loss": 0.1894158172607422, + "mean_token_accuracy": 0.934169539809227, + "num_tokens": 6712562.0, + "step": 2800 + }, + { + "entropy": 0.2648825005441904, + "epoch": 6.23658269441402, + "grad_norm": 1.2022897005081177, + "learning_rate": 2.823298604666056e-05, + "loss": 0.19440870285034179, + "mean_token_accuracy": 0.9334753274917602, + "num_tokens": 6832843.0, + "step": 2850 + }, + { + "entropy": 0.27026796594262126, + "epoch": 6.3461117196056955, + "grad_norm": 1.0150978565216064, + "learning_rate": 2.6843144663086045e-05, + "loss": 0.19671850204467772, + "mean_token_accuracy": 0.9316870296001434, + "num_tokens": 6952871.0, + "step": 2900 + }, + { + "entropy": 0.2634010723978281, + "epoch": 6.455640744797371, + "grad_norm": 1.430558681488037, + "learning_rate": 2.546930985423105e-05, + "loss": 0.19457483291625977, + "mean_token_accuracy": 0.9327938884496689, + "num_tokens": 7072067.0, + "step": 2950 + }, + { + "entropy": 0.2702385004609823, + "epoch": 6.565169769989047, + "grad_norm": 1.0556726455688477, + "learning_rate": 2.4113485187988342e-05, + "loss": 0.19928800582885742, + "mean_token_accuracy": 0.9298818710446358, + "num_tokens": 7187280.0, + "step": 3000 + }, + { + "entropy": 0.2702864905446768, + "epoch": 6.674698795180722, + "grad_norm": 1.1694364547729492, + "learning_rate": 2.2777647966688595e-05, + "loss": 0.20284730911254883, + "mean_token_accuracy": 0.9301017987728118, + "num_tokens": 7305186.0, + "step": 3050 + }, + { + "entropy": 0.2645207424461842, + "epoch": 6.784227820372399, + "grad_norm": 1.2039296627044678, + "learning_rate": 2.146374634344989e-05, + "loss": 0.1961233139038086, + "mean_token_accuracy": 0.9320311924815178, + "num_tokens": 7427803.0, + "step": 3100 + }, + { + "entropy": 0.2703990802913904, + "epoch": 6.8937568455640745, + "grad_norm": 1.122693657875061, + "learning_rate": 2.01736964810376e-05, + "loss": 0.19937246322631835, + "mean_token_accuracy": 0.9296649679541588, + "num_tokens": 7543735.0, + "step": 3150 + }, + { + "epoch": 7.0, + "eval_entropy": 0.31011983941471766, + "eval_loss": 0.7977674007415771, + "eval_mean_token_accuracy": 0.8200774944963909, + "eval_num_tokens": 7661318.0, + "eval_runtime": 95.5905, + "eval_samples_per_second": 10.482, + "eval_steps_per_second": 1.318, + "step": 3199 + }, + { + "entropy": 0.27026369738759415, + "epoch": 7.002190580503833, + "grad_norm": 0.8098449110984802, + "learning_rate": 1.8909379757378543e-05, + "loss": 0.19773469924926756, + "mean_token_accuracy": 0.9312729013688636, + "num_tokens": 7663078.0, + "step": 3200 + }, + { + "entropy": 0.21044189006090164, + "epoch": 7.11171960569551, + "grad_norm": 1.2920125722885132, + "learning_rate": 1.7672640021804126e-05, + "loss": 0.13121842384338378, + "mean_token_accuracy": 0.954786310493946, + "num_tokens": 7782363.0, + "step": 3250 + }, + { + "entropy": 0.2033486431464553, + "epoch": 7.221248630887185, + "grad_norm": 1.1921308040618896, + "learning_rate": 1.6465280906024633e-05, + "loss": 0.13039697647094728, + "mean_token_accuracy": 0.9554795321822166, + "num_tokens": 7899912.0, + "step": 3300 + }, + { + "entropy": 0.2082062489911914, + "epoch": 7.330777656078861, + "grad_norm": 1.1978024244308472, + "learning_rate": 1.5289063193755705e-05, + "loss": 0.13286386489868163, + "mean_token_accuracy": 0.9534762743115425, + "num_tokens": 8019370.0, + "step": 3350 + }, + { + "entropy": 0.2054990328848362, + "epoch": 7.4403066812705365, + "grad_norm": 1.1366864442825317, + "learning_rate": 1.4145702252833402e-05, + "loss": 0.1314125633239746, + "mean_token_accuracy": 0.9538632571697235, + "num_tokens": 8142751.0, + "step": 3400 + }, + { + "entropy": 0.20359413929283618, + "epoch": 7.549835706462212, + "grad_norm": 1.2026054859161377, + "learning_rate": 1.3036865533562794e-05, + "loss": 0.1330260467529297, + "mean_token_accuracy": 0.9539996239542962, + "num_tokens": 8260948.0, + "step": 3450 + }, + { + "entropy": 0.21044832307845354, + "epoch": 7.659364731653889, + "grad_norm": 1.3045598268508911, + "learning_rate": 1.1964170136948073e-05, + "loss": 0.13348891258239745, + "mean_token_accuracy": 0.9526105988025665, + "num_tokens": 8378334.0, + "step": 3500 + }, + { + "entropy": 0.20063834685832263, + "epoch": 7.768893756845564, + "grad_norm": 1.001644253730774, + "learning_rate": 1.0929180456351225e-05, + "loss": 0.12778966903686523, + "mean_token_accuracy": 0.955841820538044, + "num_tokens": 8503618.0, + "step": 3550 + }, + { + "entropy": 0.20864798523485661, + "epoch": 7.87842278203724, + "grad_norm": 1.0793193578720093, + "learning_rate": 9.933405896017934e-06, + "loss": 0.13261634826660157, + "mean_token_accuracy": 0.9527471828460693, + "num_tokens": 8621388.0, + "step": 3600 + }, + { + "entropy": 0.2030112524703145, + "epoch": 7.9879518072289155, + "grad_norm": 0.9351119995117188, + "learning_rate": 8.978298669798568e-06, + "loss": 0.13032340049743651, + "mean_token_accuracy": 0.953511869609356, + "num_tokens": 8743662.0, + "step": 3650 + }, + { + "epoch": 8.0, + "eval_entropy": 0.27168176760749213, + "eval_loss": 0.9082825779914856, + "eval_mean_token_accuracy": 0.8165905882441808, + "eval_num_tokens": 8755792.0, + "eval_runtime": 95.6338, + "eval_samples_per_second": 10.477, + "eval_steps_per_second": 1.318, + "step": 3656 + }, + { + "entropy": 0.17589674077250742, + "epoch": 8.096385542168674, + "grad_norm": 0.8960739374160767, + "learning_rate": 8.065251683274092e-06, + "loss": 0.09905588150024414, + "mean_token_accuracy": 0.9655855659282568, + "num_tokens": 8864043.0, + "step": 3700 + }, + { + "entropy": 0.1714504823088646, + "epoch": 8.20591456736035, + "grad_norm": 0.9031808376312256, + "learning_rate": 7.195596502375899e-06, + "loss": 0.09377479553222656, + "mean_token_accuracy": 0.9666973549127579, + "num_tokens": 8984809.0, + "step": 3750 + }, + { + "entropy": 0.16023088485002518, + "epoch": 8.315443592552027, + "grad_norm": 1.0721874237060547, + "learning_rate": 6.370601411461736e-06, + "loss": 0.09197239875793457, + "mean_token_accuracy": 0.9688060078024864, + "num_tokens": 9108507.0, + "step": 3800 + }, + { + "entropy": 0.17518360134214161, + "epoch": 8.424972617743702, + "grad_norm": 0.7766899466514587, + "learning_rate": 5.5914695636801535e-06, + "loss": 0.09632173538208008, + "mean_token_accuracy": 0.964902586042881, + "num_tokens": 9226857.0, + "step": 3850 + }, + { + "entropy": 0.16780287627130747, + "epoch": 8.534501642935378, + "grad_norm": 0.8962944149971008, + "learning_rate": 4.859337226320612e-06, + "loss": 0.09255614280700683, + "mean_token_accuracy": 0.9672093752026558, + "num_tokens": 9349892.0, + "step": 3900 + }, + { + "entropy": 0.17159098640084267, + "epoch": 8.644030668127053, + "grad_norm": 0.965328574180603, + "learning_rate": 4.175272123708463e-06, + "loss": 0.0957132911682129, + "mean_token_accuracy": 0.9660508191585541, + "num_tokens": 9464026.0, + "step": 3950 + }, + { + "entropy": 0.16975272938609123, + "epoch": 8.75355969331873, + "grad_norm": 0.7491775751113892, + "learning_rate": 3.540271880061184e-06, + "loss": 0.09802600860595703, + "mean_token_accuracy": 0.9656609424948692, + "num_tokens": 9577583.0, + "step": 4000 + }, + { + "entropy": 0.16477238953113557, + "epoch": 8.863088718510404, + "grad_norm": 1.2014434337615967, + "learning_rate": 2.9552625645771213e-06, + "loss": 0.09635594367980957, + "mean_token_accuracy": 0.966949755847454, + "num_tokens": 9694696.0, + "step": 4050 + }, + { + "entropy": 0.16196822334080935, + "epoch": 8.97261774370208, + "grad_norm": 0.9184058308601379, + "learning_rate": 2.421097340878212e-06, + "loss": 0.09237998008728027, + "mean_token_accuracy": 0.9681418687105179, + "num_tokens": 9819102.0, + "step": 4100 + }, + { + "epoch": 9.0, + "eval_entropy": 0.23909894093161538, + "eval_loss": 1.0375252962112427, + "eval_mean_token_accuracy": 0.8140787217352126, + "eval_num_tokens": 9850266.0, + "eval_runtime": 95.5931, + "eval_samples_per_second": 10.482, + "eval_steps_per_second": 1.318, + "step": 4113 + }, + { + "entropy": 0.15947704005873564, + "epoch": 9.08105147864184, + "grad_norm": 0.6882684826850891, + "learning_rate": 1.938555222776545e-06, + "loss": 0.08486066818237305, + "mean_token_accuracy": 0.9706879769912874, + "num_tokens": 9935717.0, + "step": 4150 + }, + { + "entropy": 0.14909984514117242, + "epoch": 9.190580503833516, + "grad_norm": 0.7194035649299622, + "learning_rate": 1.5083399381792327e-06, + "loss": 0.07793830871582032, + "mean_token_accuracy": 0.972977882027626, + "num_tokens": 10059424.0, + "step": 4200 + }, + { + "entropy": 0.1589351560920477, + "epoch": 9.300109529025193, + "grad_norm": 0.7463886141777039, + "learning_rate": 1.1310789027883905e-06, + "loss": 0.08115102767944336, + "mean_token_accuracy": 0.97160077303648, + "num_tokens": 10175579.0, + "step": 4250 + }, + { + "entropy": 0.1564853686466813, + "epoch": 9.409638554216867, + "grad_norm": 0.643501877784729, + "learning_rate": 8.073223050930868e-07, + "loss": 0.08317861557006836, + "mean_token_accuracy": 0.9698313820362091, + "num_tokens": 10292773.0, + "step": 4300 + }, + { + "entropy": 0.1548204604536295, + "epoch": 9.519167579408544, + "grad_norm": 0.6438949108123779, + "learning_rate": 5.375423039875474e-07, + "loss": 0.08057842254638672, + "mean_token_accuracy": 0.9718291917443276, + "num_tokens": 10412707.0, + "step": 4350 + }, + { + "entropy": 0.14797801088541745, + "epoch": 9.628696604600218, + "grad_norm": 0.7108749747276306, + "learning_rate": 3.221323401859343e-07, + "loss": 0.07771893978118896, + "mean_token_accuracy": 0.9729431614279747, + "num_tokens": 10532761.0, + "step": 4400 + }, + { + "entropy": 0.1488303004205227, + "epoch": 9.738225629791895, + "grad_norm": 0.6439398527145386, + "learning_rate": 1.6140656243777284e-07, + "loss": 0.07880829811096192, + "mean_token_accuracy": 0.9727483147382736, + "num_tokens": 10658068.0, + "step": 4450 + }, + { + "entropy": 0.15006137326359748, + "epoch": 9.847754654983571, + "grad_norm": 0.564250648021698, + "learning_rate": 5.5599369380916595e-08, + "loss": 0.07974879741668701, + "mean_token_accuracy": 0.9721924987435341, + "num_tokens": 10778905.0, + "step": 4500 + }, + { + "entropy": 0.14989942949265242, + "epoch": 9.957283680175246, + "grad_norm": 0.8616090416908264, + "learning_rate": 4.8650677001825754e-09, + "loss": 0.08046493530273438, + "mean_token_accuracy": 0.9730298516154289, + "num_tokens": 10896097.0, + "step": 4550 + }, + { + "epoch": 10.0, + "eval_entropy": 0.22838191061265886, + "eval_loss": 1.1000539064407349, + "eval_mean_token_accuracy": 0.8129113543601263, + "eval_num_tokens": 10944740.0, + "eval_runtime": 95.6219, + "eval_samples_per_second": 10.479, + "eval_steps_per_second": 1.318, + "step": 4570 + } + ], + "logging_steps": 50, + "max_steps": 4570, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.8167748538611302e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/README.md b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/adapter_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a72a4c543bf8671cb7019360179c8cabb6e5d6c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.021223473447411947, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "gate_proj", + "v_proj", + "up_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/chat_template.jinja b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/tokenizer_config.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/trainer_state.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f41f907e0765cafd70ed8e103ab0250703975d5c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test1/checkpoint-914/trainer_state.json @@ -0,0 +1,236 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 914, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.002540482878685, + "epoch": 0.10952902519167579, + "grad_norm": 0.5552461743354797, + "learning_rate": 8.109930924389919e-06, + "loss": 1.9067156982421876, + "mean_token_accuracy": 0.6026969534158707, + "num_tokens": 119060.0, + "step": 50 + }, + { + "entropy": 1.3068562260270118, + "epoch": 0.21905805038335158, + "grad_norm": 1.0465189218521118, + "learning_rate": 1.6385370643155144e-05, + "loss": 1.1655167388916015, + "mean_token_accuracy": 0.7124340075254441, + "num_tokens": 242713.0, + "step": 100 + }, + { + "entropy": 0.8696084088087082, + "epoch": 0.32858707557502737, + "grad_norm": 0.6686795949935913, + "learning_rate": 2.4660810361920367e-05, + "loss": 0.8031976318359375, + "mean_token_accuracy": 0.7754033240675926, + "num_tokens": 360160.0, + "step": 150 + }, + { + "entropy": 0.7853927117586136, + "epoch": 0.43811610076670315, + "grad_norm": 0.9354436993598938, + "learning_rate": 3.293625008068559e-05, + "loss": 0.7226885986328125, + "mean_token_accuracy": 0.791197614967823, + "num_tokens": 485403.0, + "step": 200 + }, + { + "entropy": 0.7382483741641045, + "epoch": 0.547645125958379, + "grad_norm": 0.7535398006439209, + "learning_rate": 4.121168979945081e-05, + "loss": 0.6755471801757813, + "mean_token_accuracy": 0.8054334259033203, + "num_tokens": 601578.0, + "step": 250 + }, + { + "entropy": 0.7218390592932701, + "epoch": 0.6571741511500547, + "grad_norm": 0.8160243034362793, + "learning_rate": 4.948712951821604e-05, + "loss": 0.6614183044433594, + "mean_token_accuracy": 0.8060630604624748, + "num_tokens": 713917.0, + "step": 300 + }, + { + "entropy": 0.7012957927584648, + "epoch": 0.7667031763417306, + "grad_norm": 0.7985761165618896, + "learning_rate": 5.776256923698126e-05, + "loss": 0.6415711975097657, + "mean_token_accuracy": 0.8100408402085304, + "num_tokens": 832995.0, + "step": 350 + }, + { + "entropy": 0.6763210469484329, + "epoch": 0.8762322015334063, + "grad_norm": 0.6829700469970703, + "learning_rate": 6.603800895574648e-05, + "loss": 0.6193617248535156, + "mean_token_accuracy": 0.8140933158993721, + "num_tokens": 959304.0, + "step": 400 + }, + { + "entropy": 0.6874829810857773, + "epoch": 0.9857612267250822, + "grad_norm": 0.6983632445335388, + "learning_rate": 7.43134486745117e-05, + "loss": 0.6262834167480469, + "mean_token_accuracy": 0.8163631609082223, + "num_tokens": 1078880.0, + "step": 450 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6085402492492918, + "eval_loss": 0.6388216018676758, + "eval_mean_token_accuracy": 0.8123693073552752, + "eval_num_tokens": 1094474.0, + "eval_runtime": 95.9846, + "eval_samples_per_second": 10.439, + "eval_steps_per_second": 1.313, + "step": 457 + }, + { + "entropy": 0.6471366012337232, + "epoch": 1.0941949616648412, + "grad_norm": 0.7878272533416748, + "learning_rate": 7.561806001041411e-05, + "loss": 0.589820671081543, + "mean_token_accuracy": 0.8218218798589225, + "num_tokens": 1190065.0, + "step": 500 + }, + { + "entropy": 0.6404865515232087, + "epoch": 1.203723986856517, + "grad_norm": 0.7976964712142944, + "learning_rate": 7.554418144822605e-05, + "loss": 0.5836894607543945, + "mean_token_accuracy": 0.8222010856866837, + "num_tokens": 1313079.0, + "step": 550 + }, + { + "entropy": 0.6355892798304558, + "epoch": 1.3132530120481927, + "grad_norm": 0.7428044080734253, + "learning_rate": 7.541528503116934e-05, + "loss": 0.577253189086914, + "mean_token_accuracy": 0.8260676205158234, + "num_tokens": 1428305.0, + "step": 600 + }, + { + "entropy": 0.6170084626972675, + "epoch": 1.4227820372398685, + "grad_norm": 0.6792078614234924, + "learning_rate": 7.523155873870194e-05, + "loss": 0.561871337890625, + "mean_token_accuracy": 0.8307154527306557, + "num_tokens": 1551971.0, + "step": 650 + }, + { + "entropy": 0.6222476975619793, + "epoch": 1.5323110624315444, + "grad_norm": 0.6532447934150696, + "learning_rate": 7.499327051286336e-05, + "loss": 0.5612493515014648, + "mean_token_accuracy": 0.8296262130141259, + "num_tokens": 1673401.0, + "step": 700 + }, + { + "entropy": 0.6251883202791214, + "epoch": 1.6418400876232202, + "grad_norm": 0.7107782959938049, + "learning_rate": 7.47007678675144e-05, + "loss": 0.565279769897461, + "mean_token_accuracy": 0.8299687370657921, + "num_tokens": 1788813.0, + "step": 750 + }, + { + "entropy": 0.6177104935050011, + "epoch": 1.751369112814896, + "grad_norm": 0.7850057482719421, + "learning_rate": 7.435447738153122e-05, + "loss": 0.5564990234375, + "mean_token_accuracy": 0.8291967037320137, + "num_tokens": 1911556.0, + "step": 800 + }, + { + "entropy": 0.6029443763196468, + "epoch": 1.8608981380065717, + "grad_norm": 0.7116957306861877, + "learning_rate": 7.395490407669285e-05, + "loss": 0.5501844406127929, + "mean_token_accuracy": 0.8335196697711944, + "num_tokens": 2029129.0, + "step": 850 + }, + { + "entropy": 0.6042105440795421, + "epoch": 1.9704271631982475, + "grad_norm": 0.6712159514427185, + "learning_rate": 7.350263068116955e-05, + "loss": 0.5517086029052735, + "mean_token_accuracy": 0.8321291375160217, + "num_tokens": 2155392.0, + "step": 900 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5401396741942753, + "eval_loss": 0.5797445774078369, + "eval_mean_token_accuracy": 0.8243241475688087, + "eval_num_tokens": 2188948.0, + "eval_runtime": 95.5407, + "eval_samples_per_second": 10.488, + "eval_steps_per_second": 1.319, + "step": 914 + } + ], + "logging_steps": 50, + "max_steps": 4570, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.6028715461702656e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test2/checkpoint-914/chat_template.jinja b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test2/checkpoint-914/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test2/checkpoint-914/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test2/checkpoint-914/trainer_state.json b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test2/checkpoint-914/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8189378feb3a581a883a911b8a03fa6657d3b95c --- /dev/null +++ b/systematicity_original_Estonian/Qwen3-14B-Base_systematicity_splits_original_features_train_systematicity_splits_original_features_test2/checkpoint-914/trainer_state.json @@ -0,0 +1,236 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 914, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6095534166693688, + "epoch": 0.10952902519167579, + "grad_norm": 0.6224718689918518, + "learning_rate": 4.762402719112294e-05, + "loss": 1.4966812133789062, + "mean_token_accuracy": 0.663204056173563, + "num_tokens": 119060.0, + "step": 50 + }, + { + "entropy": 0.8165125900506973, + "epoch": 0.21905805038335158, + "grad_norm": 0.614508330821991, + "learning_rate": 9.62199733045137e-05, + "loss": 0.7493247985839844, + "mean_token_accuracy": 0.7848947149515152, + "num_tokens": 242713.0, + "step": 100 + }, + { + "entropy": 0.7436758258938789, + "epoch": 0.32858707557502737, + "grad_norm": 0.48017022013664246, + "learning_rate": 0.00014481591941790446, + "loss": 0.6848310089111328, + "mean_token_accuracy": 0.8010777249932289, + "num_tokens": 360160.0, + "step": 150 + }, + { + "entropy": 0.717863453477621, + "epoch": 0.43811610076670315, + "grad_norm": 0.575162410736084, + "learning_rate": 0.00019341186553129518, + "loss": 0.6567488861083984, + "mean_token_accuracy": 0.8061749112606048, + "num_tokens": 485403.0, + "step": 200 + }, + { + "entropy": 0.6896663670241833, + "epoch": 0.547645125958379, + "grad_norm": 0.4721410572528839, + "learning_rate": 0.00024200781164468592, + "loss": 0.6327899551391601, + "mean_token_accuracy": 0.8152807226777077, + "num_tokens": 601578.0, + "step": 250 + }, + { + "entropy": 0.6908166989684105, + "epoch": 0.6571741511500547, + "grad_norm": 0.4399911165237427, + "learning_rate": 0.0002906037577580767, + "loss": 0.6312561798095703, + "mean_token_accuracy": 0.8144896909594536, + "num_tokens": 713917.0, + "step": 300 + }, + { + "entropy": 0.683870544731617, + "epoch": 0.7667031763417306, + "grad_norm": 0.4231497347354889, + "learning_rate": 0.00033919970387146745, + "loss": 0.6193135452270507, + "mean_token_accuracy": 0.815604530274868, + "num_tokens": 832995.0, + "step": 350 + }, + { + "entropy": 0.6605491453409195, + "epoch": 0.8762322015334063, + "grad_norm": 0.41835200786590576, + "learning_rate": 0.0003877956499848582, + "loss": 0.6050848770141601, + "mean_token_accuracy": 0.8192999172210693, + "num_tokens": 959304.0, + "step": 400 + }, + { + "entropy": 0.6740348136425018, + "epoch": 0.9857612267250822, + "grad_norm": 0.47750869393348694, + "learning_rate": 0.00043639159609824894, + "loss": 0.6245782852172852, + "mean_token_accuracy": 0.8173972824215889, + "num_tokens": 1078880.0, + "step": 450 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6306851827909076, + "eval_loss": 0.6308619379997253, + "eval_mean_token_accuracy": 0.8127142730213347, + "eval_num_tokens": 1094474.0, + "eval_runtime": 91.5512, + "eval_samples_per_second": 10.934, + "eval_steps_per_second": 1.376, + "step": 457 + }, + { + "entropy": 0.6464576108588113, + "epoch": 1.0941949616648412, + "grad_norm": 0.6239075660705566, + "learning_rate": 0.00044405267808700057, + "loss": 0.5859087371826172, + "mean_token_accuracy": 0.8240844417702068, + "num_tokens": 1190065.0, + "step": 500 + }, + { + "entropy": 0.647233172506094, + "epoch": 1.203723986856517, + "grad_norm": 0.5753231048583984, + "learning_rate": 0.0004436188402790972, + "loss": 0.5960317611694336, + "mean_token_accuracy": 0.8204084092378616, + "num_tokens": 1313079.0, + "step": 550 + }, + { + "entropy": 0.6426525762677193, + "epoch": 1.3132530120481927, + "grad_norm": 0.49733421206474304, + "learning_rate": 0.00044286192058052296, + "loss": 0.5911786270141601, + "mean_token_accuracy": 0.8234749832749366, + "num_tokens": 1428305.0, + "step": 600 + }, + { + "entropy": 0.6154578351974487, + "epoch": 1.4227820372398685, + "grad_norm": 0.49131613969802856, + "learning_rate": 0.00044178302286489906, + "loss": 0.5740421676635742, + "mean_token_accuracy": 0.8279985713958741, + "num_tokens": 1551971.0, + "step": 650 + }, + { + "entropy": 0.6223083969950676, + "epoch": 1.5323110624315444, + "grad_norm": 0.4566330909729004, + "learning_rate": 0.0004403837205709015, + "loss": 0.5753194046020508, + "mean_token_accuracy": 0.8258894568681717, + "num_tokens": 1673401.0, + "step": 700 + }, + { + "entropy": 0.6717074005305768, + "epoch": 1.6418400876232202, + "grad_norm": 0.5443674921989441, + "learning_rate": 0.0004386660544075954, + "loss": 0.627062759399414, + "mean_token_accuracy": 0.8198275208473206, + "num_tokens": 1788813.0, + "step": 750 + }, + { + "entropy": 0.6333334112167358, + "epoch": 1.751369112814896, + "grad_norm": 0.6672394871711731, + "learning_rate": 0.000436632529378314, + "loss": 0.576555290222168, + "mean_token_accuracy": 0.8256966322660446, + "num_tokens": 1911556.0, + "step": 800 + }, + { + "entropy": 0.6180330719053745, + "epoch": 1.8608981380065717, + "grad_norm": 0.5858656167984009, + "learning_rate": 0.00043428611112742105, + "loss": 0.5673687362670898, + "mean_token_accuracy": 0.8295214310288429, + "num_tokens": 2029129.0, + "step": 850 + }, + { + "entropy": 0.6137671455740928, + "epoch": 1.9704271631982475, + "grad_norm": 0.3748410940170288, + "learning_rate": 0.0004316302216152864, + "loss": 0.5641871261596679, + "mean_token_accuracy": 0.830936677455902, + "num_tokens": 2155392.0, + "step": 900 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5294268639787795, + "eval_loss": 0.5735377073287964, + "eval_mean_token_accuracy": 0.8277684767094869, + "eval_num_tokens": 2188948.0, + "eval_runtime": 91.2017, + "eval_samples_per_second": 10.976, + "eval_steps_per_second": 1.382, + "step": 914 + } + ], + "logging_steps": 50, + "max_steps": 4570, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.6028715461702656e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}