diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cca3c7c734f11ebc54974199c5fee1d52977d0e4 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: transformers +model_name: Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1 +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1 + +This model is a fine-tuned version of [Qwen/Qwen3-14B-Base](https://huggingface.co/Qwen/Qwen3-14B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/8nm68no9) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..40f3fa1146a457c1d13ed66ab9732bd8b455af89 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: transformers +model_name: Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2 +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2 + +This model is a fine-tuned version of [Qwen/Qwen3-14B-Base](https://huggingface.co/Qwen/Qwen3-14B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/0sjhdwt0) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..18cb627079caf6831b750ac563e2f88693dc6af5 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/trainer_state.json @@ -0,0 +1,139 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.24539877300613497, + "eval_steps": 20, + "global_step": 100, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.665608529330176e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b909474a422d3a872fed237b59945b8025b8db33 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/trainer_state.json @@ -0,0 +1,160 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.294478527607362, + "eval_steps": 20, + "global_step": 120, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.768750043840512e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e3c9e2180b5ef6b1450f3350e9a49dd6e08867dd --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/trainer_state.json @@ -0,0 +1,181 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.34355828220858897, + "eval_steps": 20, + "global_step": 140, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.864538181083136e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8224d1feb12533f40cdc8946a4ffc24041fdad88 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/trainer_state.json @@ -0,0 +1,202 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.39263803680981596, + "eval_steps": 20, + "global_step": 160, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.036288729366528e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..274d03d632212d8c5882542f1111678f861f8f30 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/trainer_state.json @@ -0,0 +1,223 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.44171779141104295, + "eval_steps": 20, + "global_step": 180, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0027645416972288e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7c54f3b54da8c9e7ea84e6dacab772adf3583d59 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/trainer_state.json @@ -0,0 +1,55 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.049079754601226995, + "eval_steps": 20, + "global_step": 20, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.122300772712448e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1781ef130e4381e9f2a1ae81fcd84c1664e4ffa4 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/trainer_state.json @@ -0,0 +1,244 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.49079754601226994, + "eval_steps": 20, + "global_step": 200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0981223285587968e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6e275e28a7022eb30826a47d3907d13ce2f88d43 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/trainer_state.json @@ -0,0 +1,265 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5398773006134969, + "eval_steps": 20, + "global_step": 220, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2097047689330688e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c5ad595146f7d35b84b94e59437d3b211bd0756f --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/trainer_state.json @@ -0,0 +1,286 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.588957055214724, + "eval_steps": 20, + "global_step": 240, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.313927085821952e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..87a23d50dbad8a86fd8ec28cbd70fd7014e6940e --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/trainer_state.json @@ -0,0 +1,307 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6380368098159509, + "eval_steps": 20, + "global_step": 260, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.4338478512447488e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..5d4b0f04459a79627a40ff32015e5d4b3a447ae0 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/trainer_state.json @@ -0,0 +1,328 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6871165644171779, + "eval_steps": 20, + "global_step": 280, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + }, + { + "entropy": 0.6182309970259666, + "epoch": 0.6871165644171779, + "grad_norm": 0.3791189193725586, + "learning_rate": 0.0002781205962525053, + "loss": 0.5778349876403809, + "mean_token_accuracy": 0.8246920928359032, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6430293495314462, + "eval_loss": 0.584140419960022, + "eval_mean_token_accuracy": 0.8250665313856942, + "eval_num_tokens": 760708.0, + "eval_runtime": 105.0023, + "eval_samples_per_second": 13.285, + "eval_steps_per_second": 1.667, + "step": 280 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.540701193027584e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..10b7cef067e5160a2aa9015225f8aa1ad0f3e6d4 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/trainer_state.json @@ -0,0 +1,349 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7361963190184049, + "eval_steps": 20, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + }, + { + "entropy": 0.6182309970259666, + "epoch": 0.6871165644171779, + "grad_norm": 0.3791189193725586, + "learning_rate": 0.0002781205962525053, + "loss": 0.5778349876403809, + "mean_token_accuracy": 0.8246920928359032, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6430293495314462, + "eval_loss": 0.584140419960022, + "eval_mean_token_accuracy": 0.8250665313856942, + "eval_num_tokens": 760708.0, + "eval_runtime": 105.0023, + "eval_samples_per_second": 13.285, + "eval_steps_per_second": 1.667, + "step": 280 + }, + { + "entropy": 0.6252344503998757, + "epoch": 0.7361963190184049, + "grad_norm": 0.3159662187099457, + "learning_rate": 0.0002670495794561784, + "loss": 0.5845423221588135, + "mean_token_accuracy": 0.8252305090427399, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6507578223092215, + "eval_loss": 0.5797898173332214, + "eval_mean_token_accuracy": 0.8279903977257865, + "eval_num_tokens": 816117.0, + "eval_runtime": 104.8929, + "eval_samples_per_second": 13.299, + "eval_steps_per_second": 1.668, + "step": 300 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6528773005574144e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f426203261d63c73ddd09ea8d3198911b9023f35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/trainer_state.json @@ -0,0 +1,370 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7852760736196319, + "eval_steps": 20, + "global_step": 320, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + }, + { + "entropy": 0.6182309970259666, + "epoch": 0.6871165644171779, + "grad_norm": 0.3791189193725586, + "learning_rate": 0.0002781205962525053, + "loss": 0.5778349876403809, + "mean_token_accuracy": 0.8246920928359032, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6430293495314462, + "eval_loss": 0.584140419960022, + "eval_mean_token_accuracy": 0.8250665313856942, + "eval_num_tokens": 760708.0, + "eval_runtime": 105.0023, + "eval_samples_per_second": 13.285, + "eval_steps_per_second": 1.667, + "step": 280 + }, + { + "entropy": 0.6252344503998757, + "epoch": 0.7361963190184049, + "grad_norm": 0.3159662187099457, + "learning_rate": 0.0002670495794561784, + "loss": 0.5845423221588135, + "mean_token_accuracy": 0.8252305090427399, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6507578223092215, + "eval_loss": 0.5797898173332214, + "eval_mean_token_accuracy": 0.8279903977257865, + "eval_num_tokens": 816117.0, + "eval_runtime": 104.8929, + "eval_samples_per_second": 13.299, + "eval_steps_per_second": 1.668, + "step": 300 + }, + { + "entropy": 0.6199106499552727, + "epoch": 0.7852760736196319, + "grad_norm": 0.33928635716438293, + "learning_rate": 0.0002552460980628558, + "loss": 0.5756847858428955, + "mean_token_accuracy": 0.8251789346337318, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6414087894984654, + "eval_loss": 0.5759831070899963, + "eval_mean_token_accuracy": 0.8279267614228385, + "eval_num_tokens": 869286.0, + "eval_runtime": 104.9796, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 320 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.7612215564468224e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..83484ffb358dd30e7c48ff423cfa9e3c5771b98e --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/trainer_state.json @@ -0,0 +1,391 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8343558282208589, + "eval_steps": 20, + "global_step": 340, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + }, + { + "entropy": 0.6182309970259666, + "epoch": 0.6871165644171779, + "grad_norm": 0.3791189193725586, + "learning_rate": 0.0002781205962525053, + "loss": 0.5778349876403809, + "mean_token_accuracy": 0.8246920928359032, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6430293495314462, + "eval_loss": 0.584140419960022, + "eval_mean_token_accuracy": 0.8250665313856942, + "eval_num_tokens": 760708.0, + "eval_runtime": 105.0023, + "eval_samples_per_second": 13.285, + "eval_steps_per_second": 1.667, + "step": 280 + }, + { + "entropy": 0.6252344503998757, + "epoch": 0.7361963190184049, + "grad_norm": 0.3159662187099457, + "learning_rate": 0.0002670495794561784, + "loss": 0.5845423221588135, + "mean_token_accuracy": 0.8252305090427399, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6507578223092215, + "eval_loss": 0.5797898173332214, + "eval_mean_token_accuracy": 0.8279903977257865, + "eval_num_tokens": 816117.0, + "eval_runtime": 104.8929, + "eval_samples_per_second": 13.299, + "eval_steps_per_second": 1.668, + "step": 300 + }, + { + "entropy": 0.6199106499552727, + "epoch": 0.7852760736196319, + "grad_norm": 0.33928635716438293, + "learning_rate": 0.0002552460980628558, + "loss": 0.5756847858428955, + "mean_token_accuracy": 0.8251789346337318, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6414087894984654, + "eval_loss": 0.5759831070899963, + "eval_mean_token_accuracy": 0.8279267614228385, + "eval_num_tokens": 869286.0, + "eval_runtime": 104.9796, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 320 + }, + { + "entropy": 0.6080872133374214, + "epoch": 0.8343558282208589, + "grad_norm": 0.2935579717159271, + "learning_rate": 0.00024279659162748857, + "loss": 0.56481032371521, + "mean_token_accuracy": 0.8329314738512039, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6243992117473057, + "eval_loss": 0.5680164694786072, + "eval_mean_token_accuracy": 0.8308778762817383, + "eval_num_tokens": 922579.0, + "eval_runtime": 104.9853, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 340 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.8758667796463616e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1b9e1ca1c56e543d57434cfbae09e08cc0324f74 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/trainer_state.json @@ -0,0 +1,412 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8834355828220859, + "eval_steps": 20, + "global_step": 360, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + }, + { + "entropy": 0.6182309970259666, + "epoch": 0.6871165644171779, + "grad_norm": 0.3791189193725586, + "learning_rate": 0.0002781205962525053, + "loss": 0.5778349876403809, + "mean_token_accuracy": 0.8246920928359032, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6430293495314462, + "eval_loss": 0.584140419960022, + "eval_mean_token_accuracy": 0.8250665313856942, + "eval_num_tokens": 760708.0, + "eval_runtime": 105.0023, + "eval_samples_per_second": 13.285, + "eval_steps_per_second": 1.667, + "step": 280 + }, + { + "entropy": 0.6252344503998757, + "epoch": 0.7361963190184049, + "grad_norm": 0.3159662187099457, + "learning_rate": 0.0002670495794561784, + "loss": 0.5845423221588135, + "mean_token_accuracy": 0.8252305090427399, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6507578223092215, + "eval_loss": 0.5797898173332214, + "eval_mean_token_accuracy": 0.8279903977257865, + "eval_num_tokens": 816117.0, + "eval_runtime": 104.8929, + "eval_samples_per_second": 13.299, + "eval_steps_per_second": 1.668, + "step": 300 + }, + { + "entropy": 0.6199106499552727, + "epoch": 0.7852760736196319, + "grad_norm": 0.33928635716438293, + "learning_rate": 0.0002552460980628558, + "loss": 0.5756847858428955, + "mean_token_accuracy": 0.8251789346337318, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6414087894984654, + "eval_loss": 0.5759831070899963, + "eval_mean_token_accuracy": 0.8279267614228385, + "eval_num_tokens": 869286.0, + "eval_runtime": 104.9796, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 320 + }, + { + "entropy": 0.6080872133374214, + "epoch": 0.8343558282208589, + "grad_norm": 0.2935579717159271, + "learning_rate": 0.00024279659162748857, + "loss": 0.56481032371521, + "mean_token_accuracy": 0.8329314738512039, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6243992117473057, + "eval_loss": 0.5680164694786072, + "eval_mean_token_accuracy": 0.8308778762817383, + "eval_num_tokens": 922579.0, + "eval_runtime": 104.9853, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 340 + }, + { + "entropy": 0.6098237752914428, + "epoch": 0.8834355828220859, + "grad_norm": 0.3914716839790344, + "learning_rate": 0.0002297922306920359, + "loss": 0.559494924545288, + "mean_token_accuracy": 0.8337442219257355, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.59963459951537, + "eval_loss": 0.5661377310752869, + "eval_mean_token_accuracy": 0.8304069665500096, + "eval_num_tokens": 977707.0, + "eval_runtime": 104.9926, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 360 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9871186553544704e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ceabb16991b44037a7d22ccd94d270a68a6aea5b --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/trainer_state.json @@ -0,0 +1,433 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9325153374233128, + "eval_steps": 20, + "global_step": 380, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + }, + { + "entropy": 0.6182309970259666, + "epoch": 0.6871165644171779, + "grad_norm": 0.3791189193725586, + "learning_rate": 0.0002781205962525053, + "loss": 0.5778349876403809, + "mean_token_accuracy": 0.8246920928359032, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6430293495314462, + "eval_loss": 0.584140419960022, + "eval_mean_token_accuracy": 0.8250665313856942, + "eval_num_tokens": 760708.0, + "eval_runtime": 105.0023, + "eval_samples_per_second": 13.285, + "eval_steps_per_second": 1.667, + "step": 280 + }, + { + "entropy": 0.6252344503998757, + "epoch": 0.7361963190184049, + "grad_norm": 0.3159662187099457, + "learning_rate": 0.0002670495794561784, + "loss": 0.5845423221588135, + "mean_token_accuracy": 0.8252305090427399, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6507578223092215, + "eval_loss": 0.5797898173332214, + "eval_mean_token_accuracy": 0.8279903977257865, + "eval_num_tokens": 816117.0, + "eval_runtime": 104.8929, + "eval_samples_per_second": 13.299, + "eval_steps_per_second": 1.668, + "step": 300 + }, + { + "entropy": 0.6199106499552727, + "epoch": 0.7852760736196319, + "grad_norm": 0.33928635716438293, + "learning_rate": 0.0002552460980628558, + "loss": 0.5756847858428955, + "mean_token_accuracy": 0.8251789346337318, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6414087894984654, + "eval_loss": 0.5759831070899963, + "eval_mean_token_accuracy": 0.8279267614228385, + "eval_num_tokens": 869286.0, + "eval_runtime": 104.9796, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 320 + }, + { + "entropy": 0.6080872133374214, + "epoch": 0.8343558282208589, + "grad_norm": 0.2935579717159271, + "learning_rate": 0.00024279659162748857, + "loss": 0.56481032371521, + "mean_token_accuracy": 0.8329314738512039, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6243992117473057, + "eval_loss": 0.5680164694786072, + "eval_mean_token_accuracy": 0.8308778762817383, + "eval_num_tokens": 922579.0, + "eval_runtime": 104.9853, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 340 + }, + { + "entropy": 0.6098237752914428, + "epoch": 0.8834355828220859, + "grad_norm": 0.3914716839790344, + "learning_rate": 0.0002297922306920359, + "loss": 0.559494924545288, + "mean_token_accuracy": 0.8337442219257355, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.59963459951537, + "eval_loss": 0.5661377310752869, + "eval_mean_token_accuracy": 0.8304069665500096, + "eval_num_tokens": 977707.0, + "eval_runtime": 104.9926, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 360 + }, + { + "entropy": 0.6145547524094581, + "epoch": 0.9325153374233128, + "grad_norm": 0.3521580696105957, + "learning_rate": 0.00021632824912303506, + "loss": 0.5705258846282959, + "mean_token_accuracy": 0.829042598605156, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.5960987096173422, + "eval_loss": 0.5600873231887817, + "eval_mean_token_accuracy": 0.8332455556733268, + "eval_num_tokens": 1028058.0, + "eval_runtime": 104.8158, + "eval_samples_per_second": 13.309, + "eval_steps_per_second": 1.67, + "step": 380 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.0914219268554752e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b36e91d890c2fc83ccfb0ce7617ac44c7d6cef5c --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/trainer_state.json @@ -0,0 +1,76 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.09815950920245399, + "eval_steps": 20, + "global_step": 40, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.304238109687808e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e81cfa3ba1f323e567c2e254130bb378062e7d04 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/trainer_state.json @@ -0,0 +1,454 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9815950920245399, + "eval_steps": 20, + "global_step": 400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + }, + { + "entropy": 0.6182309970259666, + "epoch": 0.6871165644171779, + "grad_norm": 0.3791189193725586, + "learning_rate": 0.0002781205962525053, + "loss": 0.5778349876403809, + "mean_token_accuracy": 0.8246920928359032, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6430293495314462, + "eval_loss": 0.584140419960022, + "eval_mean_token_accuracy": 0.8250665313856942, + "eval_num_tokens": 760708.0, + "eval_runtime": 105.0023, + "eval_samples_per_second": 13.285, + "eval_steps_per_second": 1.667, + "step": 280 + }, + { + "entropy": 0.6252344503998757, + "epoch": 0.7361963190184049, + "grad_norm": 0.3159662187099457, + "learning_rate": 0.0002670495794561784, + "loss": 0.5845423221588135, + "mean_token_accuracy": 0.8252305090427399, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6507578223092215, + "eval_loss": 0.5797898173332214, + "eval_mean_token_accuracy": 0.8279903977257865, + "eval_num_tokens": 816117.0, + "eval_runtime": 104.8929, + "eval_samples_per_second": 13.299, + "eval_steps_per_second": 1.668, + "step": 300 + }, + { + "entropy": 0.6199106499552727, + "epoch": 0.7852760736196319, + "grad_norm": 0.33928635716438293, + "learning_rate": 0.0002552460980628558, + "loss": 0.5756847858428955, + "mean_token_accuracy": 0.8251789346337318, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6414087894984654, + "eval_loss": 0.5759831070899963, + "eval_mean_token_accuracy": 0.8279267614228385, + "eval_num_tokens": 869286.0, + "eval_runtime": 104.9796, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 320 + }, + { + "entropy": 0.6080872133374214, + "epoch": 0.8343558282208589, + "grad_norm": 0.2935579717159271, + "learning_rate": 0.00024279659162748857, + "loss": 0.56481032371521, + "mean_token_accuracy": 0.8329314738512039, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6243992117473057, + "eval_loss": 0.5680164694786072, + "eval_mean_token_accuracy": 0.8308778762817383, + "eval_num_tokens": 922579.0, + "eval_runtime": 104.9853, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 340 + }, + { + "entropy": 0.6098237752914428, + "epoch": 0.8834355828220859, + "grad_norm": 0.3914716839790344, + "learning_rate": 0.0002297922306920359, + "loss": 0.559494924545288, + "mean_token_accuracy": 0.8337442219257355, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.59963459951537, + "eval_loss": 0.5661377310752869, + "eval_mean_token_accuracy": 0.8304069665500096, + "eval_num_tokens": 977707.0, + "eval_runtime": 104.9926, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 360 + }, + { + "entropy": 0.6145547524094581, + "epoch": 0.9325153374233128, + "grad_norm": 0.3521580696105957, + "learning_rate": 0.00021632824912303506, + "loss": 0.5705258846282959, + "mean_token_accuracy": 0.829042598605156, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.5960987096173422, + "eval_loss": 0.5600873231887817, + "eval_mean_token_accuracy": 0.8332455556733268, + "eval_num_tokens": 1028058.0, + "eval_runtime": 104.8158, + "eval_samples_per_second": 13.309, + "eval_steps_per_second": 1.67, + "step": 380 + }, + { + "entropy": 0.5970950566232205, + "epoch": 0.9815950920245399, + "grad_norm": 0.28969496488571167, + "learning_rate": 0.00020250324669252877, + "loss": 0.5550286293029785, + "mean_token_accuracy": 0.8330720633268356, + "num_tokens": 1083860.0, + "step": 400 + }, + { + "epoch": 0.9815950920245399, + "eval_entropy": 0.590186082805906, + "eval_loss": 0.5563607215881348, + "eval_mean_token_accuracy": 0.833972008228302, + "eval_num_tokens": 1083860.0, + "eval_runtime": 104.9587, + "eval_samples_per_second": 13.291, + "eval_steps_per_second": 1.667, + "step": 400 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.204630205689856e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..5d712f384e6ab0695e4ba1e1d7eba71b1b16cbe2 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/trainer_state.json @@ -0,0 +1,475 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0294478527607362, + "eval_steps": 20, + "global_step": 420, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + }, + { + "entropy": 0.6182309970259666, + "epoch": 0.6871165644171779, + "grad_norm": 0.3791189193725586, + "learning_rate": 0.0002781205962525053, + "loss": 0.5778349876403809, + "mean_token_accuracy": 0.8246920928359032, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6430293495314462, + "eval_loss": 0.584140419960022, + "eval_mean_token_accuracy": 0.8250665313856942, + "eval_num_tokens": 760708.0, + "eval_runtime": 105.0023, + "eval_samples_per_second": 13.285, + "eval_steps_per_second": 1.667, + "step": 280 + }, + { + "entropy": 0.6252344503998757, + "epoch": 0.7361963190184049, + "grad_norm": 0.3159662187099457, + "learning_rate": 0.0002670495794561784, + "loss": 0.5845423221588135, + "mean_token_accuracy": 0.8252305090427399, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6507578223092215, + "eval_loss": 0.5797898173332214, + "eval_mean_token_accuracy": 0.8279903977257865, + "eval_num_tokens": 816117.0, + "eval_runtime": 104.8929, + "eval_samples_per_second": 13.299, + "eval_steps_per_second": 1.668, + "step": 300 + }, + { + "entropy": 0.6199106499552727, + "epoch": 0.7852760736196319, + "grad_norm": 0.33928635716438293, + "learning_rate": 0.0002552460980628558, + "loss": 0.5756847858428955, + "mean_token_accuracy": 0.8251789346337318, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6414087894984654, + "eval_loss": 0.5759831070899963, + "eval_mean_token_accuracy": 0.8279267614228385, + "eval_num_tokens": 869286.0, + "eval_runtime": 104.9796, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 320 + }, + { + "entropy": 0.6080872133374214, + "epoch": 0.8343558282208589, + "grad_norm": 0.2935579717159271, + "learning_rate": 0.00024279659162748857, + "loss": 0.56481032371521, + "mean_token_accuracy": 0.8329314738512039, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6243992117473057, + "eval_loss": 0.5680164694786072, + "eval_mean_token_accuracy": 0.8308778762817383, + "eval_num_tokens": 922579.0, + "eval_runtime": 104.9853, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 340 + }, + { + "entropy": 0.6098237752914428, + "epoch": 0.8834355828220859, + "grad_norm": 0.3914716839790344, + "learning_rate": 0.0002297922306920359, + "loss": 0.559494924545288, + "mean_token_accuracy": 0.8337442219257355, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.59963459951537, + "eval_loss": 0.5661377310752869, + "eval_mean_token_accuracy": 0.8304069665500096, + "eval_num_tokens": 977707.0, + "eval_runtime": 104.9926, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 360 + }, + { + "entropy": 0.6145547524094581, + "epoch": 0.9325153374233128, + "grad_norm": 0.3521580696105957, + "learning_rate": 0.00021632824912303506, + "loss": 0.5705258846282959, + "mean_token_accuracy": 0.829042598605156, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.5960987096173422, + "eval_loss": 0.5600873231887817, + "eval_mean_token_accuracy": 0.8332455556733268, + "eval_num_tokens": 1028058.0, + "eval_runtime": 104.8158, + "eval_samples_per_second": 13.309, + "eval_steps_per_second": 1.67, + "step": 380 + }, + { + "entropy": 0.5970950566232205, + "epoch": 0.9815950920245399, + "grad_norm": 0.28969496488571167, + "learning_rate": 0.00020250324669252877, + "loss": 0.5550286293029785, + "mean_token_accuracy": 0.8330720633268356, + "num_tokens": 1083860.0, + "step": 400 + }, + { + "epoch": 0.9815950920245399, + "eval_entropy": 0.590186082805906, + "eval_loss": 0.5563607215881348, + "eval_mean_token_accuracy": 0.833972008228302, + "eval_num_tokens": 1083860.0, + "eval_runtime": 104.9587, + "eval_samples_per_second": 13.291, + "eval_steps_per_second": 1.667, + "step": 400 + }, + { + "entropy": 0.5686816084079254, + "epoch": 1.0294478527607362, + "grad_norm": 0.30075210332870483, + "learning_rate": 0.0001884184670097092, + "loss": 0.5245012760162353, + "mean_token_accuracy": 0.8398806681999793, + "num_tokens": 1136522.0, + "step": 420 + }, + { + "epoch": 1.0294478527607362, + "eval_entropy": 0.5584452276570456, + "eval_loss": 0.557037889957428, + "eval_mean_token_accuracy": 0.8341134299550738, + "eval_num_tokens": 1136522.0, + "eval_runtime": 104.9666, + "eval_samples_per_second": 13.29, + "eval_steps_per_second": 1.667, + "step": 420 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.3068699778589696e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..547215a6e90313f8b9e939d213ef8247cc319764 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.06684998388944888, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a29f7e69a745d6f9b4ddf16ae73346e039c4b5b1 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/trainer_state.json @@ -0,0 +1,297 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1167, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6304629689455032, + "epoch": 0.1287001287001287, + "grad_norm": 0.9736679196357727, + "learning_rate": 3.2321893373855725e-05, + "loss": 1.5586164855957032, + "mean_token_accuracy": 0.6625205805897713, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.865804158449173, + "epoch": 0.2574002574002574, + "grad_norm": 0.8218768239021301, + "learning_rate": 6.530341722472892e-05, + "loss": 0.7942887115478515, + "mean_token_accuracy": 0.7780012512207031, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7885943764448166, + "epoch": 0.3861003861003861, + "grad_norm": 0.6200866103172302, + "learning_rate": 9.828494107560211e-05, + "loss": 0.7239870452880859, + "mean_token_accuracy": 0.7923985701799393, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7731903672218323, + "epoch": 0.5148005148005148, + "grad_norm": 0.645903468132019, + "learning_rate": 0.00013126646492647528, + "loss": 0.7023859405517578, + "mean_token_accuracy": 0.7972224324941635, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7606095188856125, + "epoch": 0.6435006435006435, + "grad_norm": 0.6395116448402405, + "learning_rate": 0.00016424798877734848, + "loss": 0.6980245208740234, + "mean_token_accuracy": 0.8005052077770233, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.7547655135393143, + "epoch": 0.7722007722007722, + "grad_norm": 0.6048902869224548, + "learning_rate": 0.00019722951262822168, + "loss": 0.6896456146240234, + "mean_token_accuracy": 0.8002784997224808, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.750541335940361, + "epoch": 0.9009009009009009, + "grad_norm": 0.5950207114219666, + "learning_rate": 0.00023021103647909486, + "loss": 0.6888568878173829, + "mean_token_accuracy": 0.8017277073860168, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6049834216378399, + "eval_loss": 0.6702247858047485, + "eval_mean_token_accuracy": 0.8023447219243983, + "eval_num_tokens": 555493.0, + "eval_runtime": 160.7595, + "eval_samples_per_second": 9.617, + "eval_steps_per_second": 1.207, + "step": 389 + }, + { + "entropy": 0.7378247476587392, + "epoch": 1.0283140283140284, + "grad_norm": 0.7072293758392334, + "learning_rate": 0.0002565910901719884, + "loss": 0.6729380798339843, + "mean_token_accuracy": 0.8071240547931555, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7119287192821503, + "epoch": 1.157014157014157, + "grad_norm": 0.8078219294548035, + "learning_rate": 0.00025641034526723873, + "loss": 0.6499625396728516, + "mean_token_accuracy": 0.8119793623685837, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7278847670555115, + "epoch": 1.2857142857142856, + "grad_norm": 0.6917271018028259, + "learning_rate": 0.0002559717467472163, + "loss": 0.6674861145019532, + "mean_token_accuracy": 0.8067185115814209, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7150312864780426, + "epoch": 1.4144144144144144, + "grad_norm": 0.8094608187675476, + "learning_rate": 0.0002552761773865406, + "loss": 0.6532902526855469, + "mean_token_accuracy": 0.8110801106691361, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6913726192712784, + "epoch": 1.5431145431145432, + "grad_norm": 0.6806851625442505, + "learning_rate": 0.0002543250371693649, + "loss": 0.6355234527587891, + "mean_token_accuracy": 0.8154943329095841, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.716510910987854, + "epoch": 1.6718146718146718, + "grad_norm": 0.8052911162376404, + "learning_rate": 0.00025312024047160467, + "loss": 0.6559846496582031, + "mean_token_accuracy": 0.8098146516084671, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6818760082125663, + "epoch": 1.8005148005148004, + "grad_norm": 0.5995563268661499, + "learning_rate": 0.0002516642122078408, + "loss": 0.6270184326171875, + "mean_token_accuracy": 0.8169869929552078, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6728256559371948, + "epoch": 1.9292149292149292, + "grad_norm": 0.684234082698822, + "learning_rate": 0.00024995988295065347, + "loss": 0.622582130432129, + "mean_token_accuracy": 0.8187594223022461, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5827852641184306, + "eval_loss": 0.6126551628112793, + "eval_mean_token_accuracy": 0.8199613155777922, + "eval_num_tokens": 1110986.0, + "eval_runtime": 160.3137, + "eval_samples_per_second": 9.644, + "eval_steps_per_second": 1.21, + "step": 778 + }, + { + "entropy": 0.6285555227236315, + "epoch": 2.056628056628057, + "grad_norm": 0.729573130607605, + "learning_rate": 0.00024801068303220925, + "loss": 0.5709608840942383, + "mean_token_accuracy": 0.8302884445045934, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5837715390324593, + "epoch": 2.1853281853281854, + "grad_norm": 0.657953143119812, + "learning_rate": 0.00024582053563997405, + "loss": 0.5185433197021484, + "mean_token_accuracy": 0.8406957858800888, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.5731439217925072, + "epoch": 2.314028314028314, + "grad_norm": 0.7121743559837341, + "learning_rate": 0.00024339384892044734, + "loss": 0.5192452621459961, + "mean_token_accuracy": 0.8415917128324508, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5781797507405281, + "epoch": 2.4427284427284426, + "grad_norm": 0.583035409450531, + "learning_rate": 0.00024073550710681184, + "loss": 0.51892822265625, + "mean_token_accuracy": 0.8394862592220307, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5838576689362526, + "epoch": 2.571428571428571, + "grad_norm": 0.7375105619430542, + "learning_rate": 0.00023785086068835518, + "loss": 0.5248625946044921, + "mean_token_accuracy": 0.8392664766311646, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5801274874806404, + "epoch": 2.7001287001287, + "grad_norm": 0.5255026817321777, + "learning_rate": 0.00023474571564145, + "loss": 0.5241841125488281, + "mean_token_accuracy": 0.8405563312768937, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.582165597975254, + "epoch": 2.828828828828829, + "grad_norm": 0.6157832145690918, + "learning_rate": 0.00023142632174376774, + "loss": 0.5229883956909179, + "mean_token_accuracy": 0.8406760370731354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5778958490490913, + "epoch": 2.9575289575289574, + "grad_norm": 0.5259988307952881, + "learning_rate": 0.000227899359995246, + "loss": 0.5213600540161133, + "mean_token_accuracy": 0.8410600781440735, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5146706245609165, + "eval_loss": 0.6217455267906189, + "eval_mean_token_accuracy": 0.8225257052588708, + "eval_num_tokens": 1666479.0, + "eval_runtime": 159.8044, + "eval_samples_per_second": 9.674, + "eval_steps_per_second": 1.214, + "step": 1167 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.7917443648582656e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..547215a6e90313f8b9e939d213ef8247cc319764 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.06684998388944888, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..07040b332ade8c73f3be5a2c3de32e72e5f20403 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/trainer_state.json @@ -0,0 +1,388 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1556, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6304629689455032, + "epoch": 0.1287001287001287, + "grad_norm": 0.9736679196357727, + "learning_rate": 3.2321893373855725e-05, + "loss": 1.5586164855957032, + "mean_token_accuracy": 0.6625205805897713, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.865804158449173, + "epoch": 0.2574002574002574, + "grad_norm": 0.8218768239021301, + "learning_rate": 6.530341722472892e-05, + "loss": 0.7942887115478515, + "mean_token_accuracy": 0.7780012512207031, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7885943764448166, + "epoch": 0.3861003861003861, + "grad_norm": 0.6200866103172302, + "learning_rate": 9.828494107560211e-05, + "loss": 0.7239870452880859, + "mean_token_accuracy": 0.7923985701799393, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7731903672218323, + "epoch": 0.5148005148005148, + "grad_norm": 0.645903468132019, + "learning_rate": 0.00013126646492647528, + "loss": 0.7023859405517578, + "mean_token_accuracy": 0.7972224324941635, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7606095188856125, + "epoch": 0.6435006435006435, + "grad_norm": 0.6395116448402405, + "learning_rate": 0.00016424798877734848, + "loss": 0.6980245208740234, + "mean_token_accuracy": 0.8005052077770233, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.7547655135393143, + "epoch": 0.7722007722007722, + "grad_norm": 0.6048902869224548, + "learning_rate": 0.00019722951262822168, + "loss": 0.6896456146240234, + "mean_token_accuracy": 0.8002784997224808, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.750541335940361, + "epoch": 0.9009009009009009, + "grad_norm": 0.5950207114219666, + "learning_rate": 0.00023021103647909486, + "loss": 0.6888568878173829, + "mean_token_accuracy": 0.8017277073860168, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6049834216378399, + "eval_loss": 0.6702247858047485, + "eval_mean_token_accuracy": 0.8023447219243983, + "eval_num_tokens": 555493.0, + "eval_runtime": 160.7595, + "eval_samples_per_second": 9.617, + "eval_steps_per_second": 1.207, + "step": 389 + }, + { + "entropy": 0.7378247476587392, + "epoch": 1.0283140283140284, + "grad_norm": 0.7072293758392334, + "learning_rate": 0.0002565910901719884, + "loss": 0.6729380798339843, + "mean_token_accuracy": 0.8071240547931555, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7119287192821503, + "epoch": 1.157014157014157, + "grad_norm": 0.8078219294548035, + "learning_rate": 0.00025641034526723873, + "loss": 0.6499625396728516, + "mean_token_accuracy": 0.8119793623685837, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7278847670555115, + "epoch": 1.2857142857142856, + "grad_norm": 0.6917271018028259, + "learning_rate": 0.0002559717467472163, + "loss": 0.6674861145019532, + "mean_token_accuracy": 0.8067185115814209, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7150312864780426, + "epoch": 1.4144144144144144, + "grad_norm": 0.8094608187675476, + "learning_rate": 0.0002552761773865406, + "loss": 0.6532902526855469, + "mean_token_accuracy": 0.8110801106691361, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6913726192712784, + "epoch": 1.5431145431145432, + "grad_norm": 0.6806851625442505, + "learning_rate": 0.0002543250371693649, + "loss": 0.6355234527587891, + "mean_token_accuracy": 0.8154943329095841, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.716510910987854, + "epoch": 1.6718146718146718, + "grad_norm": 0.8052911162376404, + "learning_rate": 0.00025312024047160467, + "loss": 0.6559846496582031, + "mean_token_accuracy": 0.8098146516084671, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6818760082125663, + "epoch": 1.8005148005148004, + "grad_norm": 0.5995563268661499, + "learning_rate": 0.0002516642122078408, + "loss": 0.6270184326171875, + "mean_token_accuracy": 0.8169869929552078, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6728256559371948, + "epoch": 1.9292149292149292, + "grad_norm": 0.684234082698822, + "learning_rate": 0.00024995988295065347, + "loss": 0.622582130432129, + "mean_token_accuracy": 0.8187594223022461, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5827852641184306, + "eval_loss": 0.6126551628112793, + "eval_mean_token_accuracy": 0.8199613155777922, + "eval_num_tokens": 1110986.0, + "eval_runtime": 160.3137, + "eval_samples_per_second": 9.644, + "eval_steps_per_second": 1.21, + "step": 778 + }, + { + "entropy": 0.6285555227236315, + "epoch": 2.056628056628057, + "grad_norm": 0.729573130607605, + "learning_rate": 0.00024801068303220925, + "loss": 0.5709608840942383, + "mean_token_accuracy": 0.8302884445045934, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5837715390324593, + "epoch": 2.1853281853281854, + "grad_norm": 0.657953143119812, + "learning_rate": 0.00024582053563997405, + "loss": 0.5185433197021484, + "mean_token_accuracy": 0.8406957858800888, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.5731439217925072, + "epoch": 2.314028314028314, + "grad_norm": 0.7121743559837341, + "learning_rate": 0.00024339384892044734, + "loss": 0.5192452621459961, + "mean_token_accuracy": 0.8415917128324508, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5781797507405281, + "epoch": 2.4427284427284426, + "grad_norm": 0.583035409450531, + "learning_rate": 0.00024073550710681184, + "loss": 0.51892822265625, + "mean_token_accuracy": 0.8394862592220307, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5838576689362526, + "epoch": 2.571428571428571, + "grad_norm": 0.7375105619430542, + "learning_rate": 0.00023785086068835518, + "loss": 0.5248625946044921, + "mean_token_accuracy": 0.8392664766311646, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5801274874806404, + "epoch": 2.7001287001287, + "grad_norm": 0.5255026817321777, + "learning_rate": 0.00023474571564145, + "loss": 0.5241841125488281, + "mean_token_accuracy": 0.8405563312768937, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.582165597975254, + "epoch": 2.828828828828829, + "grad_norm": 0.6157832145690918, + "learning_rate": 0.00023142632174376774, + "loss": 0.5229883956909179, + "mean_token_accuracy": 0.8406760370731354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5778958490490913, + "epoch": 2.9575289575289574, + "grad_norm": 0.5259988307952881, + "learning_rate": 0.000227899359995246, + "loss": 0.5213600540161133, + "mean_token_accuracy": 0.8410600781440735, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5146706245609165, + "eval_loss": 0.6217455267906189, + "eval_mean_token_accuracy": 0.8225257052588708, + "eval_num_tokens": 1666479.0, + "eval_runtime": 159.8044, + "eval_samples_per_second": 9.674, + "eval_steps_per_second": 1.214, + "step": 1167 + }, + { + "entropy": 0.5129444800844096, + "epoch": 3.0849420849420848, + "grad_norm": 0.6443549394607544, + "learning_rate": 0.00022417192917112742, + "loss": 0.4500067520141602, + "mean_token_accuracy": 0.8588406985456293, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.45547543197870255, + "epoch": 3.213642213642214, + "grad_norm": 0.6051533818244934, + "learning_rate": 0.0002202515315341355, + "loss": 0.3920443344116211, + "mean_token_accuracy": 0.8705952340364456, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4667788457870483, + "epoch": 3.3423423423423424, + "grad_norm": 0.625999927520752, + "learning_rate": 0.0002161460577345441, + "loss": 0.4082052230834961, + "mean_token_accuracy": 0.8685764318704605, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.49247278362512587, + "epoch": 3.471042471042471, + "grad_norm": 0.7345328330993652, + "learning_rate": 0.000211863770928533, + "loss": 0.4317153549194336, + "mean_token_accuracy": 0.8622037881612777, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.48875897347927094, + "epoch": 3.5997425997425996, + "grad_norm": 0.6730979681015015, + "learning_rate": 0.00020741329014679433, + "loss": 0.4286677932739258, + "mean_token_accuracy": 0.8626715314388275, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.4849738264083862, + "epoch": 3.7284427284427286, + "grad_norm": 0.6888502240180969, + "learning_rate": 0.00020280357294686466, + "loss": 0.4235057830810547, + "mean_token_accuracy": 0.8637587743997573, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.4883710837364197, + "epoch": 3.857142857142857, + "grad_norm": 0.6455050110816956, + "learning_rate": 0.00019804389738409812, + "loss": 0.42961734771728516, + "mean_token_accuracy": 0.8626390981674195, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.46666738361120225, + "epoch": 3.985842985842986, + "grad_norm": 0.6892870664596558, + "learning_rate": 0.0001931438433375683, + "loss": 0.4121334457397461, + "mean_token_accuracy": 0.8682902973890304, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4674856377016638, + "eval_loss": 0.6489306688308716, + "eval_mean_token_accuracy": 0.8197588545759928, + "eval_num_tokens": 2221972.0, + "eval_runtime": 159.7918, + "eval_samples_per_second": 9.675, + "eval_steps_per_second": 1.214, + "step": 1556 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.723284547240653e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..547215a6e90313f8b9e939d213ef8247cc319764 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.06684998388944888, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..978d48578d70c810ede64a198bce59229f50ae8d --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/trainer_state.json @@ -0,0 +1,469 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 1945, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6304629689455032, + "epoch": 0.1287001287001287, + "grad_norm": 0.9736679196357727, + "learning_rate": 3.2321893373855725e-05, + "loss": 1.5586164855957032, + "mean_token_accuracy": 0.6625205805897713, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.865804158449173, + "epoch": 0.2574002574002574, + "grad_norm": 0.8218768239021301, + "learning_rate": 6.530341722472892e-05, + "loss": 0.7942887115478515, + "mean_token_accuracy": 0.7780012512207031, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7885943764448166, + "epoch": 0.3861003861003861, + "grad_norm": 0.6200866103172302, + "learning_rate": 9.828494107560211e-05, + "loss": 0.7239870452880859, + "mean_token_accuracy": 0.7923985701799393, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7731903672218323, + "epoch": 0.5148005148005148, + "grad_norm": 0.645903468132019, + "learning_rate": 0.00013126646492647528, + "loss": 0.7023859405517578, + "mean_token_accuracy": 0.7972224324941635, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7606095188856125, + "epoch": 0.6435006435006435, + "grad_norm": 0.6395116448402405, + "learning_rate": 0.00016424798877734848, + "loss": 0.6980245208740234, + "mean_token_accuracy": 0.8005052077770233, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.7547655135393143, + "epoch": 0.7722007722007722, + "grad_norm": 0.6048902869224548, + "learning_rate": 0.00019722951262822168, + "loss": 0.6896456146240234, + "mean_token_accuracy": 0.8002784997224808, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.750541335940361, + "epoch": 0.9009009009009009, + "grad_norm": 0.5950207114219666, + "learning_rate": 0.00023021103647909486, + "loss": 0.6888568878173829, + "mean_token_accuracy": 0.8017277073860168, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6049834216378399, + "eval_loss": 0.6702247858047485, + "eval_mean_token_accuracy": 0.8023447219243983, + "eval_num_tokens": 555493.0, + "eval_runtime": 160.7595, + "eval_samples_per_second": 9.617, + "eval_steps_per_second": 1.207, + "step": 389 + }, + { + "entropy": 0.7378247476587392, + "epoch": 1.0283140283140284, + "grad_norm": 0.7072293758392334, + "learning_rate": 0.0002565910901719884, + "loss": 0.6729380798339843, + "mean_token_accuracy": 0.8071240547931555, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7119287192821503, + "epoch": 1.157014157014157, + "grad_norm": 0.8078219294548035, + "learning_rate": 0.00025641034526723873, + "loss": 0.6499625396728516, + "mean_token_accuracy": 0.8119793623685837, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7278847670555115, + "epoch": 1.2857142857142856, + "grad_norm": 0.6917271018028259, + "learning_rate": 0.0002559717467472163, + "loss": 0.6674861145019532, + "mean_token_accuracy": 0.8067185115814209, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7150312864780426, + "epoch": 1.4144144144144144, + "grad_norm": 0.8094608187675476, + "learning_rate": 0.0002552761773865406, + "loss": 0.6532902526855469, + "mean_token_accuracy": 0.8110801106691361, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6913726192712784, + "epoch": 1.5431145431145432, + "grad_norm": 0.6806851625442505, + "learning_rate": 0.0002543250371693649, + "loss": 0.6355234527587891, + "mean_token_accuracy": 0.8154943329095841, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.716510910987854, + "epoch": 1.6718146718146718, + "grad_norm": 0.8052911162376404, + "learning_rate": 0.00025312024047160467, + "loss": 0.6559846496582031, + "mean_token_accuracy": 0.8098146516084671, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6818760082125663, + "epoch": 1.8005148005148004, + "grad_norm": 0.5995563268661499, + "learning_rate": 0.0002516642122078408, + "loss": 0.6270184326171875, + "mean_token_accuracy": 0.8169869929552078, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6728256559371948, + "epoch": 1.9292149292149292, + "grad_norm": 0.684234082698822, + "learning_rate": 0.00024995988295065347, + "loss": 0.622582130432129, + "mean_token_accuracy": 0.8187594223022461, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5827852641184306, + "eval_loss": 0.6126551628112793, + "eval_mean_token_accuracy": 0.8199613155777922, + "eval_num_tokens": 1110986.0, + "eval_runtime": 160.3137, + "eval_samples_per_second": 9.644, + "eval_steps_per_second": 1.21, + "step": 778 + }, + { + "entropy": 0.6285555227236315, + "epoch": 2.056628056628057, + "grad_norm": 0.729573130607605, + "learning_rate": 0.00024801068303220925, + "loss": 0.5709608840942383, + "mean_token_accuracy": 0.8302884445045934, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5837715390324593, + "epoch": 2.1853281853281854, + "grad_norm": 0.657953143119812, + "learning_rate": 0.00024582053563997405, + "loss": 0.5185433197021484, + "mean_token_accuracy": 0.8406957858800888, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.5731439217925072, + "epoch": 2.314028314028314, + "grad_norm": 0.7121743559837341, + "learning_rate": 0.00024339384892044734, + "loss": 0.5192452621459961, + "mean_token_accuracy": 0.8415917128324508, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5781797507405281, + "epoch": 2.4427284427284426, + "grad_norm": 0.583035409450531, + "learning_rate": 0.00024073550710681184, + "loss": 0.51892822265625, + "mean_token_accuracy": 0.8394862592220307, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5838576689362526, + "epoch": 2.571428571428571, + "grad_norm": 0.7375105619430542, + "learning_rate": 0.00023785086068835518, + "loss": 0.5248625946044921, + "mean_token_accuracy": 0.8392664766311646, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5801274874806404, + "epoch": 2.7001287001287, + "grad_norm": 0.5255026817321777, + "learning_rate": 0.00023474571564145, + "loss": 0.5241841125488281, + "mean_token_accuracy": 0.8405563312768937, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.582165597975254, + "epoch": 2.828828828828829, + "grad_norm": 0.6157832145690918, + "learning_rate": 0.00023142632174376774, + "loss": 0.5229883956909179, + "mean_token_accuracy": 0.8406760370731354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5778958490490913, + "epoch": 2.9575289575289574, + "grad_norm": 0.5259988307952881, + "learning_rate": 0.000227899359995246, + "loss": 0.5213600540161133, + "mean_token_accuracy": 0.8410600781440735, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5146706245609165, + "eval_loss": 0.6217455267906189, + "eval_mean_token_accuracy": 0.8225257052588708, + "eval_num_tokens": 1666479.0, + "eval_runtime": 159.8044, + "eval_samples_per_second": 9.674, + "eval_steps_per_second": 1.214, + "step": 1167 + }, + { + "entropy": 0.5129444800844096, + "epoch": 3.0849420849420848, + "grad_norm": 0.6443549394607544, + "learning_rate": 0.00022417192917112742, + "loss": 0.4500067520141602, + "mean_token_accuracy": 0.8588406985456293, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.45547543197870255, + "epoch": 3.213642213642214, + "grad_norm": 0.6051533818244934, + "learning_rate": 0.0002202515315341355, + "loss": 0.3920443344116211, + "mean_token_accuracy": 0.8705952340364456, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4667788457870483, + "epoch": 3.3423423423423424, + "grad_norm": 0.625999927520752, + "learning_rate": 0.0002161460577345441, + "loss": 0.4082052230834961, + "mean_token_accuracy": 0.8685764318704605, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.49247278362512587, + "epoch": 3.471042471042471, + "grad_norm": 0.7345328330993652, + "learning_rate": 0.000211863770928533, + "loss": 0.4317153549194336, + "mean_token_accuracy": 0.8622037881612777, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.48875897347927094, + "epoch": 3.5997425997425996, + "grad_norm": 0.6730979681015015, + "learning_rate": 0.00020741329014679433, + "loss": 0.4286677932739258, + "mean_token_accuracy": 0.8626715314388275, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.4849738264083862, + "epoch": 3.7284427284427286, + "grad_norm": 0.6888502240180969, + "learning_rate": 0.00020280357294686466, + "loss": 0.4235057830810547, + "mean_token_accuracy": 0.8637587743997573, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.4883710837364197, + "epoch": 3.857142857142857, + "grad_norm": 0.6455050110816956, + "learning_rate": 0.00019804389738409812, + "loss": 0.42961734771728516, + "mean_token_accuracy": 0.8626390981674195, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.46666738361120225, + "epoch": 3.985842985842986, + "grad_norm": 0.6892870664596558, + "learning_rate": 0.0001931438433375683, + "loss": 0.4121334457397461, + "mean_token_accuracy": 0.8682902973890304, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4674856377016638, + "eval_loss": 0.6489306688308716, + "eval_mean_token_accuracy": 0.8197588545759928, + "eval_num_tokens": 2221972.0, + "eval_runtime": 159.7918, + "eval_samples_per_second": 9.675, + "eval_steps_per_second": 1.214, + "step": 1556 + }, + { + "entropy": 0.3744586084527199, + "epoch": 4.113256113256114, + "grad_norm": 0.6487554311752319, + "learning_rate": 0.00018811327322848455, + "loss": 0.31006853103637694, + "mean_token_accuracy": 0.8973448577553335, + "num_tokens": 2282164.0, + "step": 1600 + }, + { + "entropy": 0.3579921764135361, + "epoch": 4.241956241956242, + "grad_norm": 0.6199011206626892, + "learning_rate": 0.00018296231216993105, + "loss": 0.2979485511779785, + "mean_token_accuracy": 0.8998763573169708, + "num_tokens": 2353213.0, + "step": 1650 + }, + { + "entropy": 0.3664910554885864, + "epoch": 4.370656370656371, + "grad_norm": 0.7049311399459839, + "learning_rate": 0.00017770132758788103, + "loss": 0.3122232818603516, + "mean_token_accuracy": 0.8964972496032715, + "num_tokens": 2425068.0, + "step": 1700 + }, + { + "entropy": 0.35544642210006716, + "epoch": 4.499356499356499, + "grad_norm": 0.6574566960334778, + "learning_rate": 0.00017234090835450445, + "loss": 0.3042313766479492, + "mean_token_accuracy": 0.8966764956712723, + "num_tokens": 2498321.0, + "step": 1750 + }, + { + "entropy": 0.35768423914909364, + "epoch": 4.628056628056628, + "grad_norm": 0.6159058809280396, + "learning_rate": 0.00016689184347576657, + "loss": 0.30206684112548826, + "mean_token_accuracy": 0.898514216542244, + "num_tokens": 2570321.0, + "step": 1800 + }, + { + "entropy": 0.37273951262235644, + "epoch": 4.756756756756757, + "grad_norm": 0.5503262281417847, + "learning_rate": 0.00016136510037621443, + "loss": 0.31558765411376954, + "mean_token_accuracy": 0.8939771527051925, + "num_tokens": 2642851.0, + "step": 1850 + }, + { + "entropy": 0.3542595499753952, + "epoch": 4.885456885456885, + "grad_norm": 0.7454004883766174, + "learning_rate": 0.0001557718028246568, + "loss": 0.3020258712768555, + "mean_token_accuracy": 0.8977061492204667, + "num_tokens": 2715629.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3610583918610799, + "eval_loss": 0.7293093204498291, + "eval_mean_token_accuracy": 0.8165675112881612, + "eval_num_tokens": 2777465.0, + "eval_runtime": 160.2029, + "eval_samples_per_second": 9.65, + "eval_steps_per_second": 1.211, + "step": 1945 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.653233039031091e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..547215a6e90313f8b9e939d213ef8247cc319764 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.06684998388944888, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f5a550b06b2ae0a0324c6668961f25d46e46787e --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/trainer_state.json @@ -0,0 +1,560 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2334, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6304629689455032, + "epoch": 0.1287001287001287, + "grad_norm": 0.9736679196357727, + "learning_rate": 3.2321893373855725e-05, + "loss": 1.5586164855957032, + "mean_token_accuracy": 0.6625205805897713, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.865804158449173, + "epoch": 0.2574002574002574, + "grad_norm": 0.8218768239021301, + "learning_rate": 6.530341722472892e-05, + "loss": 0.7942887115478515, + "mean_token_accuracy": 0.7780012512207031, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7885943764448166, + "epoch": 0.3861003861003861, + "grad_norm": 0.6200866103172302, + "learning_rate": 9.828494107560211e-05, + "loss": 0.7239870452880859, + "mean_token_accuracy": 0.7923985701799393, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7731903672218323, + "epoch": 0.5148005148005148, + "grad_norm": 0.645903468132019, + "learning_rate": 0.00013126646492647528, + "loss": 0.7023859405517578, + "mean_token_accuracy": 0.7972224324941635, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7606095188856125, + "epoch": 0.6435006435006435, + "grad_norm": 0.6395116448402405, + "learning_rate": 0.00016424798877734848, + "loss": 0.6980245208740234, + "mean_token_accuracy": 0.8005052077770233, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.7547655135393143, + "epoch": 0.7722007722007722, + "grad_norm": 0.6048902869224548, + "learning_rate": 0.00019722951262822168, + "loss": 0.6896456146240234, + "mean_token_accuracy": 0.8002784997224808, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.750541335940361, + "epoch": 0.9009009009009009, + "grad_norm": 0.5950207114219666, + "learning_rate": 0.00023021103647909486, + "loss": 0.6888568878173829, + "mean_token_accuracy": 0.8017277073860168, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6049834216378399, + "eval_loss": 0.6702247858047485, + "eval_mean_token_accuracy": 0.8023447219243983, + "eval_num_tokens": 555493.0, + "eval_runtime": 160.7595, + "eval_samples_per_second": 9.617, + "eval_steps_per_second": 1.207, + "step": 389 + }, + { + "entropy": 0.7378247476587392, + "epoch": 1.0283140283140284, + "grad_norm": 0.7072293758392334, + "learning_rate": 0.0002565910901719884, + "loss": 0.6729380798339843, + "mean_token_accuracy": 0.8071240547931555, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7119287192821503, + "epoch": 1.157014157014157, + "grad_norm": 0.8078219294548035, + "learning_rate": 0.00025641034526723873, + "loss": 0.6499625396728516, + "mean_token_accuracy": 0.8119793623685837, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7278847670555115, + "epoch": 1.2857142857142856, + "grad_norm": 0.6917271018028259, + "learning_rate": 0.0002559717467472163, + "loss": 0.6674861145019532, + "mean_token_accuracy": 0.8067185115814209, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7150312864780426, + "epoch": 1.4144144144144144, + "grad_norm": 0.8094608187675476, + "learning_rate": 0.0002552761773865406, + "loss": 0.6532902526855469, + "mean_token_accuracy": 0.8110801106691361, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6913726192712784, + "epoch": 1.5431145431145432, + "grad_norm": 0.6806851625442505, + "learning_rate": 0.0002543250371693649, + "loss": 0.6355234527587891, + "mean_token_accuracy": 0.8154943329095841, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.716510910987854, + "epoch": 1.6718146718146718, + "grad_norm": 0.8052911162376404, + "learning_rate": 0.00025312024047160467, + "loss": 0.6559846496582031, + "mean_token_accuracy": 0.8098146516084671, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6818760082125663, + "epoch": 1.8005148005148004, + "grad_norm": 0.5995563268661499, + "learning_rate": 0.0002516642122078408, + "loss": 0.6270184326171875, + "mean_token_accuracy": 0.8169869929552078, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6728256559371948, + "epoch": 1.9292149292149292, + "grad_norm": 0.684234082698822, + "learning_rate": 0.00024995988295065347, + "loss": 0.622582130432129, + "mean_token_accuracy": 0.8187594223022461, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5827852641184306, + "eval_loss": 0.6126551628112793, + "eval_mean_token_accuracy": 0.8199613155777922, + "eval_num_tokens": 1110986.0, + "eval_runtime": 160.3137, + "eval_samples_per_second": 9.644, + "eval_steps_per_second": 1.21, + "step": 778 + }, + { + "entropy": 0.6285555227236315, + "epoch": 2.056628056628057, + "grad_norm": 0.729573130607605, + "learning_rate": 0.00024801068303220925, + "loss": 0.5709608840942383, + "mean_token_accuracy": 0.8302884445045934, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5837715390324593, + "epoch": 2.1853281853281854, + "grad_norm": 0.657953143119812, + "learning_rate": 0.00024582053563997405, + "loss": 0.5185433197021484, + "mean_token_accuracy": 0.8406957858800888, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.5731439217925072, + "epoch": 2.314028314028314, + "grad_norm": 0.7121743559837341, + "learning_rate": 0.00024339384892044734, + "loss": 0.5192452621459961, + "mean_token_accuracy": 0.8415917128324508, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5781797507405281, + "epoch": 2.4427284427284426, + "grad_norm": 0.583035409450531, + "learning_rate": 0.00024073550710681184, + "loss": 0.51892822265625, + "mean_token_accuracy": 0.8394862592220307, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5838576689362526, + "epoch": 2.571428571428571, + "grad_norm": 0.7375105619430542, + "learning_rate": 0.00023785086068835518, + "loss": 0.5248625946044921, + "mean_token_accuracy": 0.8392664766311646, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5801274874806404, + "epoch": 2.7001287001287, + "grad_norm": 0.5255026817321777, + "learning_rate": 0.00023474571564145, + "loss": 0.5241841125488281, + "mean_token_accuracy": 0.8405563312768937, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.582165597975254, + "epoch": 2.828828828828829, + "grad_norm": 0.6157832145690918, + "learning_rate": 0.00023142632174376774, + "loss": 0.5229883956909179, + "mean_token_accuracy": 0.8406760370731354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5778958490490913, + "epoch": 2.9575289575289574, + "grad_norm": 0.5259988307952881, + "learning_rate": 0.000227899359995246, + "loss": 0.5213600540161133, + "mean_token_accuracy": 0.8410600781440735, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5146706245609165, + "eval_loss": 0.6217455267906189, + "eval_mean_token_accuracy": 0.8225257052588708, + "eval_num_tokens": 1666479.0, + "eval_runtime": 159.8044, + "eval_samples_per_second": 9.674, + "eval_steps_per_second": 1.214, + "step": 1167 + }, + { + "entropy": 0.5129444800844096, + "epoch": 3.0849420849420848, + "grad_norm": 0.6443549394607544, + "learning_rate": 0.00022417192917112742, + "loss": 0.4500067520141602, + "mean_token_accuracy": 0.8588406985456293, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.45547543197870255, + "epoch": 3.213642213642214, + "grad_norm": 0.6051533818244934, + "learning_rate": 0.0002202515315341355, + "loss": 0.3920443344116211, + "mean_token_accuracy": 0.8705952340364456, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4667788457870483, + "epoch": 3.3423423423423424, + "grad_norm": 0.625999927520752, + "learning_rate": 0.0002161460577345441, + "loss": 0.4082052230834961, + "mean_token_accuracy": 0.8685764318704605, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.49247278362512587, + "epoch": 3.471042471042471, + "grad_norm": 0.7345328330993652, + "learning_rate": 0.000211863770928533, + "loss": 0.4317153549194336, + "mean_token_accuracy": 0.8622037881612777, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.48875897347927094, + "epoch": 3.5997425997425996, + "grad_norm": 0.6730979681015015, + "learning_rate": 0.00020741329014679433, + "loss": 0.4286677932739258, + "mean_token_accuracy": 0.8626715314388275, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.4849738264083862, + "epoch": 3.7284427284427286, + "grad_norm": 0.6888502240180969, + "learning_rate": 0.00020280357294686466, + "loss": 0.4235057830810547, + "mean_token_accuracy": 0.8637587743997573, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.4883710837364197, + "epoch": 3.857142857142857, + "grad_norm": 0.6455050110816956, + "learning_rate": 0.00019804389738409812, + "loss": 0.42961734771728516, + "mean_token_accuracy": 0.8626390981674195, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.46666738361120225, + "epoch": 3.985842985842986, + "grad_norm": 0.6892870664596558, + "learning_rate": 0.0001931438433375683, + "loss": 0.4121334457397461, + "mean_token_accuracy": 0.8682902973890304, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4674856377016638, + "eval_loss": 0.6489306688308716, + "eval_mean_token_accuracy": 0.8197588545759928, + "eval_num_tokens": 2221972.0, + "eval_runtime": 159.7918, + "eval_samples_per_second": 9.675, + "eval_steps_per_second": 1.214, + "step": 1556 + }, + { + "entropy": 0.3744586084527199, + "epoch": 4.113256113256114, + "grad_norm": 0.6487554311752319, + "learning_rate": 0.00018811327322848455, + "loss": 0.31006853103637694, + "mean_token_accuracy": 0.8973448577553335, + "num_tokens": 2282164.0, + "step": 1600 + }, + { + "entropy": 0.3579921764135361, + "epoch": 4.241956241956242, + "grad_norm": 0.6199011206626892, + "learning_rate": 0.00018296231216993105, + "loss": 0.2979485511779785, + "mean_token_accuracy": 0.8998763573169708, + "num_tokens": 2353213.0, + "step": 1650 + }, + { + "entropy": 0.3664910554885864, + "epoch": 4.370656370656371, + "grad_norm": 0.7049311399459839, + "learning_rate": 0.00017770132758788103, + "loss": 0.3122232818603516, + "mean_token_accuracy": 0.8964972496032715, + "num_tokens": 2425068.0, + "step": 1700 + }, + { + "entropy": 0.35544642210006716, + "epoch": 4.499356499356499, + "grad_norm": 0.6574566960334778, + "learning_rate": 0.00017234090835450445, + "loss": 0.3042313766479492, + "mean_token_accuracy": 0.8966764956712723, + "num_tokens": 2498321.0, + "step": 1750 + }, + { + "entropy": 0.35768423914909364, + "epoch": 4.628056628056628, + "grad_norm": 0.6159058809280396, + "learning_rate": 0.00016689184347576657, + "loss": 0.30206684112548826, + "mean_token_accuracy": 0.898514216542244, + "num_tokens": 2570321.0, + "step": 1800 + }, + { + "entropy": 0.37273951262235644, + "epoch": 4.756756756756757, + "grad_norm": 0.5503262281417847, + "learning_rate": 0.00016136510037621443, + "loss": 0.31558765411376954, + "mean_token_accuracy": 0.8939771527051925, + "num_tokens": 2642851.0, + "step": 1850 + }, + { + "entropy": 0.3542595499753952, + "epoch": 4.885456885456885, + "grad_norm": 0.7454004883766174, + "learning_rate": 0.0001557718028246568, + "loss": 0.3020258712768555, + "mean_token_accuracy": 0.8977061492204667, + "num_tokens": 2715629.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3610583918610799, + "eval_loss": 0.7293093204498291, + "eval_mean_token_accuracy": 0.8165675112881612, + "eval_num_tokens": 2777465.0, + "eval_runtime": 160.2029, + "eval_samples_per_second": 9.65, + "eval_steps_per_second": 1.211, + "step": 1945 + }, + { + "entropy": 0.3579569302424036, + "epoch": 5.012870012870013, + "grad_norm": 0.4158821403980255, + "learning_rate": 0.00015012320854516744, + "loss": 0.3004540252685547, + "mean_token_accuracy": 0.8991928124668622, + "num_tokens": 2784896.0, + "step": 1950 + }, + { + "entropy": 0.26410215690732003, + "epoch": 5.141570141570142, + "grad_norm": 0.5636745095252991, + "learning_rate": 0.00014443068655847486, + "loss": 0.20726581573486327, + "mean_token_accuracy": 0.9306374716758729, + "num_tokens": 2854374.0, + "step": 2000 + }, + { + "entropy": 0.26181382283568383, + "epoch": 5.27027027027027, + "grad_norm": 0.5340386033058167, + "learning_rate": 0.00013870569429934264, + "loss": 0.2055559730529785, + "mean_token_accuracy": 0.9296587175130844, + "num_tokens": 2927362.0, + "step": 2050 + }, + { + "entropy": 0.261084942817688, + "epoch": 5.398970398970399, + "grad_norm": 0.6604517102241516, + "learning_rate": 0.00013295975455599807, + "loss": 0.20649850845336915, + "mean_token_accuracy": 0.9292166239023208, + "num_tokens": 3000143.0, + "step": 2100 + }, + { + "entropy": 0.2605279567837715, + "epoch": 5.527670527670527, + "grad_norm": 0.6845375895500183, + "learning_rate": 0.00012720443227802214, + "loss": 0.20826065063476562, + "mean_token_accuracy": 0.9290810519456864, + "num_tokens": 3072561.0, + "step": 2150 + }, + { + "entropy": 0.2661572727560997, + "epoch": 5.656370656370656, + "grad_norm": 0.557294487953186, + "learning_rate": 0.00012145131129938128, + "loss": 0.2132364273071289, + "mean_token_accuracy": 0.9271187698841095, + "num_tokens": 3144831.0, + "step": 2200 + }, + { + "entropy": 0.2664845323562622, + "epoch": 5.785070785070785, + "grad_norm": 0.5024778842926025, + "learning_rate": 0.00011571197102344961, + "loss": 0.21299808502197265, + "mean_token_accuracy": 0.9263874608278274, + "num_tokens": 3215960.0, + "step": 2250 + }, + { + "entropy": 0.26299254536628724, + "epoch": 5.913770913770914, + "grad_norm": 0.634398341178894, + "learning_rate": 0.00010999796311694913, + "loss": 0.20765880584716798, + "mean_token_accuracy": 0.928708678483963, + "num_tokens": 3287499.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.26795706152915955, + "eval_loss": 0.8664970993995667, + "eval_mean_token_accuracy": 0.8188820490517568, + "eval_num_tokens": 3332958.0, + "eval_runtime": 160.1105, + "eval_samples_per_second": 9.656, + "eval_steps_per_second": 1.212, + "step": 2334 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.587061113467187e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..547215a6e90313f8b9e939d213ef8247cc319764 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.06684998388944888, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..54771e1de85c916e9e58331540f16d15a705ed44 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/trainer_state.json @@ -0,0 +1,651 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2723, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6304629689455032, + "epoch": 0.1287001287001287, + "grad_norm": 0.9736679196357727, + "learning_rate": 3.2321893373855725e-05, + "loss": 1.5586164855957032, + "mean_token_accuracy": 0.6625205805897713, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.865804158449173, + "epoch": 0.2574002574002574, + "grad_norm": 0.8218768239021301, + "learning_rate": 6.530341722472892e-05, + "loss": 0.7942887115478515, + "mean_token_accuracy": 0.7780012512207031, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7885943764448166, + "epoch": 0.3861003861003861, + "grad_norm": 0.6200866103172302, + "learning_rate": 9.828494107560211e-05, + "loss": 0.7239870452880859, + "mean_token_accuracy": 0.7923985701799393, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7731903672218323, + "epoch": 0.5148005148005148, + "grad_norm": 0.645903468132019, + "learning_rate": 0.00013126646492647528, + "loss": 0.7023859405517578, + "mean_token_accuracy": 0.7972224324941635, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7606095188856125, + "epoch": 0.6435006435006435, + "grad_norm": 0.6395116448402405, + "learning_rate": 0.00016424798877734848, + "loss": 0.6980245208740234, + "mean_token_accuracy": 0.8005052077770233, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.7547655135393143, + "epoch": 0.7722007722007722, + "grad_norm": 0.6048902869224548, + "learning_rate": 0.00019722951262822168, + "loss": 0.6896456146240234, + "mean_token_accuracy": 0.8002784997224808, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.750541335940361, + "epoch": 0.9009009009009009, + "grad_norm": 0.5950207114219666, + "learning_rate": 0.00023021103647909486, + "loss": 0.6888568878173829, + "mean_token_accuracy": 0.8017277073860168, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6049834216378399, + "eval_loss": 0.6702247858047485, + "eval_mean_token_accuracy": 0.8023447219243983, + "eval_num_tokens": 555493.0, + "eval_runtime": 160.7595, + "eval_samples_per_second": 9.617, + "eval_steps_per_second": 1.207, + "step": 389 + }, + { + "entropy": 0.7378247476587392, + "epoch": 1.0283140283140284, + "grad_norm": 0.7072293758392334, + "learning_rate": 0.0002565910901719884, + "loss": 0.6729380798339843, + "mean_token_accuracy": 0.8071240547931555, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7119287192821503, + "epoch": 1.157014157014157, + "grad_norm": 0.8078219294548035, + "learning_rate": 0.00025641034526723873, + "loss": 0.6499625396728516, + "mean_token_accuracy": 0.8119793623685837, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7278847670555115, + "epoch": 1.2857142857142856, + "grad_norm": 0.6917271018028259, + "learning_rate": 0.0002559717467472163, + "loss": 0.6674861145019532, + "mean_token_accuracy": 0.8067185115814209, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7150312864780426, + "epoch": 1.4144144144144144, + "grad_norm": 0.8094608187675476, + "learning_rate": 0.0002552761773865406, + "loss": 0.6532902526855469, + "mean_token_accuracy": 0.8110801106691361, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6913726192712784, + "epoch": 1.5431145431145432, + "grad_norm": 0.6806851625442505, + "learning_rate": 0.0002543250371693649, + "loss": 0.6355234527587891, + "mean_token_accuracy": 0.8154943329095841, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.716510910987854, + "epoch": 1.6718146718146718, + "grad_norm": 0.8052911162376404, + "learning_rate": 0.00025312024047160467, + "loss": 0.6559846496582031, + "mean_token_accuracy": 0.8098146516084671, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6818760082125663, + "epoch": 1.8005148005148004, + "grad_norm": 0.5995563268661499, + "learning_rate": 0.0002516642122078408, + "loss": 0.6270184326171875, + "mean_token_accuracy": 0.8169869929552078, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6728256559371948, + "epoch": 1.9292149292149292, + "grad_norm": 0.684234082698822, + "learning_rate": 0.00024995988295065347, + "loss": 0.622582130432129, + "mean_token_accuracy": 0.8187594223022461, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5827852641184306, + "eval_loss": 0.6126551628112793, + "eval_mean_token_accuracy": 0.8199613155777922, + "eval_num_tokens": 1110986.0, + "eval_runtime": 160.3137, + "eval_samples_per_second": 9.644, + "eval_steps_per_second": 1.21, + "step": 778 + }, + { + "entropy": 0.6285555227236315, + "epoch": 2.056628056628057, + "grad_norm": 0.729573130607605, + "learning_rate": 0.00024801068303220925, + "loss": 0.5709608840942383, + "mean_token_accuracy": 0.8302884445045934, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5837715390324593, + "epoch": 2.1853281853281854, + "grad_norm": 0.657953143119812, + "learning_rate": 0.00024582053563997405, + "loss": 0.5185433197021484, + "mean_token_accuracy": 0.8406957858800888, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.5731439217925072, + "epoch": 2.314028314028314, + "grad_norm": 0.7121743559837341, + "learning_rate": 0.00024339384892044734, + "loss": 0.5192452621459961, + "mean_token_accuracy": 0.8415917128324508, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5781797507405281, + "epoch": 2.4427284427284426, + "grad_norm": 0.583035409450531, + "learning_rate": 0.00024073550710681184, + "loss": 0.51892822265625, + "mean_token_accuracy": 0.8394862592220307, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5838576689362526, + "epoch": 2.571428571428571, + "grad_norm": 0.7375105619430542, + "learning_rate": 0.00023785086068835518, + "loss": 0.5248625946044921, + "mean_token_accuracy": 0.8392664766311646, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5801274874806404, + "epoch": 2.7001287001287, + "grad_norm": 0.5255026817321777, + "learning_rate": 0.00023474571564145, + "loss": 0.5241841125488281, + "mean_token_accuracy": 0.8405563312768937, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.582165597975254, + "epoch": 2.828828828828829, + "grad_norm": 0.6157832145690918, + "learning_rate": 0.00023142632174376774, + "loss": 0.5229883956909179, + "mean_token_accuracy": 0.8406760370731354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5778958490490913, + "epoch": 2.9575289575289574, + "grad_norm": 0.5259988307952881, + "learning_rate": 0.000227899359995246, + "loss": 0.5213600540161133, + "mean_token_accuracy": 0.8410600781440735, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5146706245609165, + "eval_loss": 0.6217455267906189, + "eval_mean_token_accuracy": 0.8225257052588708, + "eval_num_tokens": 1666479.0, + "eval_runtime": 159.8044, + "eval_samples_per_second": 9.674, + "eval_steps_per_second": 1.214, + "step": 1167 + }, + { + "entropy": 0.5129444800844096, + "epoch": 3.0849420849420848, + "grad_norm": 0.6443549394607544, + "learning_rate": 0.00022417192917112742, + "loss": 0.4500067520141602, + "mean_token_accuracy": 0.8588406985456293, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.45547543197870255, + "epoch": 3.213642213642214, + "grad_norm": 0.6051533818244934, + "learning_rate": 0.0002202515315341355, + "loss": 0.3920443344116211, + "mean_token_accuracy": 0.8705952340364456, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4667788457870483, + "epoch": 3.3423423423423424, + "grad_norm": 0.625999927520752, + "learning_rate": 0.0002161460577345441, + "loss": 0.4082052230834961, + "mean_token_accuracy": 0.8685764318704605, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.49247278362512587, + "epoch": 3.471042471042471, + "grad_norm": 0.7345328330993652, + "learning_rate": 0.000211863770928533, + "loss": 0.4317153549194336, + "mean_token_accuracy": 0.8622037881612777, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.48875897347927094, + "epoch": 3.5997425997425996, + "grad_norm": 0.6730979681015015, + "learning_rate": 0.00020741329014679433, + "loss": 0.4286677932739258, + "mean_token_accuracy": 0.8626715314388275, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.4849738264083862, + "epoch": 3.7284427284427286, + "grad_norm": 0.6888502240180969, + "learning_rate": 0.00020280357294686466, + "loss": 0.4235057830810547, + "mean_token_accuracy": 0.8637587743997573, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.4883710837364197, + "epoch": 3.857142857142857, + "grad_norm": 0.6455050110816956, + "learning_rate": 0.00019804389738409812, + "loss": 0.42961734771728516, + "mean_token_accuracy": 0.8626390981674195, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.46666738361120225, + "epoch": 3.985842985842986, + "grad_norm": 0.6892870664596558, + "learning_rate": 0.0001931438433375683, + "loss": 0.4121334457397461, + "mean_token_accuracy": 0.8682902973890304, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4674856377016638, + "eval_loss": 0.6489306688308716, + "eval_mean_token_accuracy": 0.8197588545759928, + "eval_num_tokens": 2221972.0, + "eval_runtime": 159.7918, + "eval_samples_per_second": 9.675, + "eval_steps_per_second": 1.214, + "step": 1556 + }, + { + "entropy": 0.3744586084527199, + "epoch": 4.113256113256114, + "grad_norm": 0.6487554311752319, + "learning_rate": 0.00018811327322848455, + "loss": 0.31006853103637694, + "mean_token_accuracy": 0.8973448577553335, + "num_tokens": 2282164.0, + "step": 1600 + }, + { + "entropy": 0.3579921764135361, + "epoch": 4.241956241956242, + "grad_norm": 0.6199011206626892, + "learning_rate": 0.00018296231216993105, + "loss": 0.2979485511779785, + "mean_token_accuracy": 0.8998763573169708, + "num_tokens": 2353213.0, + "step": 1650 + }, + { + "entropy": 0.3664910554885864, + "epoch": 4.370656370656371, + "grad_norm": 0.7049311399459839, + "learning_rate": 0.00017770132758788103, + "loss": 0.3122232818603516, + "mean_token_accuracy": 0.8964972496032715, + "num_tokens": 2425068.0, + "step": 1700 + }, + { + "entropy": 0.35544642210006716, + "epoch": 4.499356499356499, + "grad_norm": 0.6574566960334778, + "learning_rate": 0.00017234090835450445, + "loss": 0.3042313766479492, + "mean_token_accuracy": 0.8966764956712723, + "num_tokens": 2498321.0, + "step": 1750 + }, + { + "entropy": 0.35768423914909364, + "epoch": 4.628056628056628, + "grad_norm": 0.6159058809280396, + "learning_rate": 0.00016689184347576657, + "loss": 0.30206684112548826, + "mean_token_accuracy": 0.898514216542244, + "num_tokens": 2570321.0, + "step": 1800 + }, + { + "entropy": 0.37273951262235644, + "epoch": 4.756756756756757, + "grad_norm": 0.5503262281417847, + "learning_rate": 0.00016136510037621443, + "loss": 0.31558765411376954, + "mean_token_accuracy": 0.8939771527051925, + "num_tokens": 2642851.0, + "step": 1850 + }, + { + "entropy": 0.3542595499753952, + "epoch": 4.885456885456885, + "grad_norm": 0.7454004883766174, + "learning_rate": 0.0001557718028246568, + "loss": 0.3020258712768555, + "mean_token_accuracy": 0.8977061492204667, + "num_tokens": 2715629.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3610583918610799, + "eval_loss": 0.7293093204498291, + "eval_mean_token_accuracy": 0.8165675112881612, + "eval_num_tokens": 2777465.0, + "eval_runtime": 160.2029, + "eval_samples_per_second": 9.65, + "eval_steps_per_second": 1.211, + "step": 1945 + }, + { + "entropy": 0.3579569302424036, + "epoch": 5.012870012870013, + "grad_norm": 0.4158821403980255, + "learning_rate": 0.00015012320854516744, + "loss": 0.3004540252685547, + "mean_token_accuracy": 0.8991928124668622, + "num_tokens": 2784896.0, + "step": 1950 + }, + { + "entropy": 0.26410215690732003, + "epoch": 5.141570141570142, + "grad_norm": 0.5636745095252991, + "learning_rate": 0.00014443068655847486, + "loss": 0.20726581573486327, + "mean_token_accuracy": 0.9306374716758729, + "num_tokens": 2854374.0, + "step": 2000 + }, + { + "entropy": 0.26181382283568383, + "epoch": 5.27027027027027, + "grad_norm": 0.5340386033058167, + "learning_rate": 0.00013870569429934264, + "loss": 0.2055559730529785, + "mean_token_accuracy": 0.9296587175130844, + "num_tokens": 2927362.0, + "step": 2050 + }, + { + "entropy": 0.261084942817688, + "epoch": 5.398970398970399, + "grad_norm": 0.6604517102241516, + "learning_rate": 0.00013295975455599807, + "loss": 0.20649850845336915, + "mean_token_accuracy": 0.9292166239023208, + "num_tokens": 3000143.0, + "step": 2100 + }, + { + "entropy": 0.2605279567837715, + "epoch": 5.527670527670527, + "grad_norm": 0.6845375895500183, + "learning_rate": 0.00012720443227802214, + "loss": 0.20826065063476562, + "mean_token_accuracy": 0.9290810519456864, + "num_tokens": 3072561.0, + "step": 2150 + }, + { + "entropy": 0.2661572727560997, + "epoch": 5.656370656370656, + "grad_norm": 0.557294487953186, + "learning_rate": 0.00012145131129938128, + "loss": 0.2132364273071289, + "mean_token_accuracy": 0.9271187698841095, + "num_tokens": 3144831.0, + "step": 2200 + }, + { + "entropy": 0.2664845323562622, + "epoch": 5.785070785070785, + "grad_norm": 0.5024778842926025, + "learning_rate": 0.00011571197102344961, + "loss": 0.21299808502197265, + "mean_token_accuracy": 0.9263874608278274, + "num_tokens": 3215960.0, + "step": 2250 + }, + { + "entropy": 0.26299254536628724, + "epoch": 5.913770913770914, + "grad_norm": 0.634398341178894, + "learning_rate": 0.00010999796311694913, + "loss": 0.20765880584716798, + "mean_token_accuracy": 0.928708678483963, + "num_tokens": 3287499.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.26795706152915955, + "eval_loss": 0.8664970993995667, + "eval_mean_token_accuracy": 0.8188820490517568, + "eval_num_tokens": 3332958.0, + "eval_runtime": 160.1105, + "eval_samples_per_second": 9.656, + "eval_steps_per_second": 1.212, + "step": 2334 + }, + { + "entropy": 0.2456816165435194, + "epoch": 6.041184041184041, + "grad_norm": 0.638988196849823, + "learning_rate": 0.00010432078825971574, + "loss": 0.18737899780273437, + "mean_token_accuracy": 0.9367745241733513, + "num_tokens": 3356879.0, + "step": 2350 + }, + { + "entropy": 0.19167753487825392, + "epoch": 6.1698841698841695, + "grad_norm": 0.6846708059310913, + "learning_rate": 9.869187299708746e-05, + "loss": 0.1415392589569092, + "mean_token_accuracy": 0.9528085774183274, + "num_tokens": 3427587.0, + "step": 2400 + }, + { + "entropy": 0.19594975739717482, + "epoch": 6.298584298584299, + "grad_norm": 0.45718589425086975, + "learning_rate": 9.312254674150445e-05, + "loss": 0.14171995162963869, + "mean_token_accuracy": 0.9512906008958817, + "num_tokens": 3498688.0, + "step": 2450 + }, + { + "entropy": 0.20213496513664722, + "epoch": 6.427284427284428, + "grad_norm": 0.9218644499778748, + "learning_rate": 8.762401896960985e-05, + "loss": 0.14789142608642578, + "mean_token_accuracy": 0.9499747276306152, + "num_tokens": 3566137.0, + "step": 2500 + }, + { + "entropy": 0.18978311866521835, + "epoch": 6.555984555984556, + "grad_norm": 0.3909820318222046, + "learning_rate": 8.22073566607478e-05, + "loss": 0.1412898349761963, + "mean_token_accuracy": 0.951915112733841, + "num_tokens": 3639573.0, + "step": 2550 + }, + { + "entropy": 0.19396651685237884, + "epoch": 6.684684684684685, + "grad_norm": 0.633430540561676, + "learning_rate": 7.688346202226782e-05, + "loss": 0.14628902435302735, + "mean_token_accuracy": 0.9509830921888351, + "num_tokens": 3710433.0, + "step": 2600 + }, + { + "entropy": 0.19417933478951455, + "epoch": 6.813384813384813, + "grad_norm": 0.4060302972793579, + "learning_rate": 7.166305054646867e-05, + "loss": 0.14396389961242675, + "mean_token_accuracy": 0.9514253032207489, + "num_tokens": 3781550.0, + "step": 2650 + }, + { + "entropy": 0.19293704330921174, + "epoch": 6.942084942084942, + "grad_norm": 0.40285640954971313, + "learning_rate": 6.655662944334679e-05, + "loss": 0.14185072898864745, + "mean_token_accuracy": 0.951703981757164, + "num_tokens": 3854952.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.20662364801487973, + "eval_loss": 1.0061800479888916, + "eval_mean_token_accuracy": 0.8215526999886503, + "eval_num_tokens": 3888451.0, + "eval_runtime": 159.9122, + "eval_samples_per_second": 9.668, + "eval_steps_per_second": 1.213, + "step": 2723 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.516671077296845e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..547215a6e90313f8b9e939d213ef8247cc319764 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.06684998388944888, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3beabea80d1eb802b56519b3c8b457f86ba38c57 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/trainer_state.json @@ -0,0 +1,742 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3112, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6304629689455032, + "epoch": 0.1287001287001287, + "grad_norm": 0.9736679196357727, + "learning_rate": 3.2321893373855725e-05, + "loss": 1.5586164855957032, + "mean_token_accuracy": 0.6625205805897713, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.865804158449173, + "epoch": 0.2574002574002574, + "grad_norm": 0.8218768239021301, + "learning_rate": 6.530341722472892e-05, + "loss": 0.7942887115478515, + "mean_token_accuracy": 0.7780012512207031, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7885943764448166, + "epoch": 0.3861003861003861, + "grad_norm": 0.6200866103172302, + "learning_rate": 9.828494107560211e-05, + "loss": 0.7239870452880859, + "mean_token_accuracy": 0.7923985701799393, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7731903672218323, + "epoch": 0.5148005148005148, + "grad_norm": 0.645903468132019, + "learning_rate": 0.00013126646492647528, + "loss": 0.7023859405517578, + "mean_token_accuracy": 0.7972224324941635, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7606095188856125, + "epoch": 0.6435006435006435, + "grad_norm": 0.6395116448402405, + "learning_rate": 0.00016424798877734848, + "loss": 0.6980245208740234, + "mean_token_accuracy": 0.8005052077770233, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.7547655135393143, + "epoch": 0.7722007722007722, + "grad_norm": 0.6048902869224548, + "learning_rate": 0.00019722951262822168, + "loss": 0.6896456146240234, + "mean_token_accuracy": 0.8002784997224808, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.750541335940361, + "epoch": 0.9009009009009009, + "grad_norm": 0.5950207114219666, + "learning_rate": 0.00023021103647909486, + "loss": 0.6888568878173829, + "mean_token_accuracy": 0.8017277073860168, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6049834216378399, + "eval_loss": 0.6702247858047485, + "eval_mean_token_accuracy": 0.8023447219243983, + "eval_num_tokens": 555493.0, + "eval_runtime": 160.7595, + "eval_samples_per_second": 9.617, + "eval_steps_per_second": 1.207, + "step": 389 + }, + { + "entropy": 0.7378247476587392, + "epoch": 1.0283140283140284, + "grad_norm": 0.7072293758392334, + "learning_rate": 0.0002565910901719884, + "loss": 0.6729380798339843, + "mean_token_accuracy": 0.8071240547931555, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7119287192821503, + "epoch": 1.157014157014157, + "grad_norm": 0.8078219294548035, + "learning_rate": 0.00025641034526723873, + "loss": 0.6499625396728516, + "mean_token_accuracy": 0.8119793623685837, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7278847670555115, + "epoch": 1.2857142857142856, + "grad_norm": 0.6917271018028259, + "learning_rate": 0.0002559717467472163, + "loss": 0.6674861145019532, + "mean_token_accuracy": 0.8067185115814209, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7150312864780426, + "epoch": 1.4144144144144144, + "grad_norm": 0.8094608187675476, + "learning_rate": 0.0002552761773865406, + "loss": 0.6532902526855469, + "mean_token_accuracy": 0.8110801106691361, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6913726192712784, + "epoch": 1.5431145431145432, + "grad_norm": 0.6806851625442505, + "learning_rate": 0.0002543250371693649, + "loss": 0.6355234527587891, + "mean_token_accuracy": 0.8154943329095841, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.716510910987854, + "epoch": 1.6718146718146718, + "grad_norm": 0.8052911162376404, + "learning_rate": 0.00025312024047160467, + "loss": 0.6559846496582031, + "mean_token_accuracy": 0.8098146516084671, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6818760082125663, + "epoch": 1.8005148005148004, + "grad_norm": 0.5995563268661499, + "learning_rate": 0.0002516642122078408, + "loss": 0.6270184326171875, + "mean_token_accuracy": 0.8169869929552078, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6728256559371948, + "epoch": 1.9292149292149292, + "grad_norm": 0.684234082698822, + "learning_rate": 0.00024995988295065347, + "loss": 0.622582130432129, + "mean_token_accuracy": 0.8187594223022461, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5827852641184306, + "eval_loss": 0.6126551628112793, + "eval_mean_token_accuracy": 0.8199613155777922, + "eval_num_tokens": 1110986.0, + "eval_runtime": 160.3137, + "eval_samples_per_second": 9.644, + "eval_steps_per_second": 1.21, + "step": 778 + }, + { + "entropy": 0.6285555227236315, + "epoch": 2.056628056628057, + "grad_norm": 0.729573130607605, + "learning_rate": 0.00024801068303220925, + "loss": 0.5709608840942383, + "mean_token_accuracy": 0.8302884445045934, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5837715390324593, + "epoch": 2.1853281853281854, + "grad_norm": 0.657953143119812, + "learning_rate": 0.00024582053563997405, + "loss": 0.5185433197021484, + "mean_token_accuracy": 0.8406957858800888, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.5731439217925072, + "epoch": 2.314028314028314, + "grad_norm": 0.7121743559837341, + "learning_rate": 0.00024339384892044734, + "loss": 0.5192452621459961, + "mean_token_accuracy": 0.8415917128324508, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5781797507405281, + "epoch": 2.4427284427284426, + "grad_norm": 0.583035409450531, + "learning_rate": 0.00024073550710681184, + "loss": 0.51892822265625, + "mean_token_accuracy": 0.8394862592220307, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5838576689362526, + "epoch": 2.571428571428571, + "grad_norm": 0.7375105619430542, + "learning_rate": 0.00023785086068835518, + "loss": 0.5248625946044921, + "mean_token_accuracy": 0.8392664766311646, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5801274874806404, + "epoch": 2.7001287001287, + "grad_norm": 0.5255026817321777, + "learning_rate": 0.00023474571564145, + "loss": 0.5241841125488281, + "mean_token_accuracy": 0.8405563312768937, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.582165597975254, + "epoch": 2.828828828828829, + "grad_norm": 0.6157832145690918, + "learning_rate": 0.00023142632174376774, + "loss": 0.5229883956909179, + "mean_token_accuracy": 0.8406760370731354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5778958490490913, + "epoch": 2.9575289575289574, + "grad_norm": 0.5259988307952881, + "learning_rate": 0.000227899359995246, + "loss": 0.5213600540161133, + "mean_token_accuracy": 0.8410600781440735, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5146706245609165, + "eval_loss": 0.6217455267906189, + "eval_mean_token_accuracy": 0.8225257052588708, + "eval_num_tokens": 1666479.0, + "eval_runtime": 159.8044, + "eval_samples_per_second": 9.674, + "eval_steps_per_second": 1.214, + "step": 1167 + }, + { + "entropy": 0.5129444800844096, + "epoch": 3.0849420849420848, + "grad_norm": 0.6443549394607544, + "learning_rate": 0.00022417192917112742, + "loss": 0.4500067520141602, + "mean_token_accuracy": 0.8588406985456293, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.45547543197870255, + "epoch": 3.213642213642214, + "grad_norm": 0.6051533818244934, + "learning_rate": 0.0002202515315341355, + "loss": 0.3920443344116211, + "mean_token_accuracy": 0.8705952340364456, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4667788457870483, + "epoch": 3.3423423423423424, + "grad_norm": 0.625999927520752, + "learning_rate": 0.0002161460577345441, + "loss": 0.4082052230834961, + "mean_token_accuracy": 0.8685764318704605, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.49247278362512587, + "epoch": 3.471042471042471, + "grad_norm": 0.7345328330993652, + "learning_rate": 0.000211863770928533, + "loss": 0.4317153549194336, + "mean_token_accuracy": 0.8622037881612777, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.48875897347927094, + "epoch": 3.5997425997425996, + "grad_norm": 0.6730979681015015, + "learning_rate": 0.00020741329014679433, + "loss": 0.4286677932739258, + "mean_token_accuracy": 0.8626715314388275, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.4849738264083862, + "epoch": 3.7284427284427286, + "grad_norm": 0.6888502240180969, + "learning_rate": 0.00020280357294686466, + "loss": 0.4235057830810547, + "mean_token_accuracy": 0.8637587743997573, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.4883710837364197, + "epoch": 3.857142857142857, + "grad_norm": 0.6455050110816956, + "learning_rate": 0.00019804389738409812, + "loss": 0.42961734771728516, + "mean_token_accuracy": 0.8626390981674195, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.46666738361120225, + "epoch": 3.985842985842986, + "grad_norm": 0.6892870664596558, + "learning_rate": 0.0001931438433375683, + "loss": 0.4121334457397461, + "mean_token_accuracy": 0.8682902973890304, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4674856377016638, + "eval_loss": 0.6489306688308716, + "eval_mean_token_accuracy": 0.8197588545759928, + "eval_num_tokens": 2221972.0, + "eval_runtime": 159.7918, + "eval_samples_per_second": 9.675, + "eval_steps_per_second": 1.214, + "step": 1556 + }, + { + "entropy": 0.3744586084527199, + "epoch": 4.113256113256114, + "grad_norm": 0.6487554311752319, + "learning_rate": 0.00018811327322848455, + "loss": 0.31006853103637694, + "mean_token_accuracy": 0.8973448577553335, + "num_tokens": 2282164.0, + "step": 1600 + }, + { + "entropy": 0.3579921764135361, + "epoch": 4.241956241956242, + "grad_norm": 0.6199011206626892, + "learning_rate": 0.00018296231216993105, + "loss": 0.2979485511779785, + "mean_token_accuracy": 0.8998763573169708, + "num_tokens": 2353213.0, + "step": 1650 + }, + { + "entropy": 0.3664910554885864, + "epoch": 4.370656370656371, + "grad_norm": 0.7049311399459839, + "learning_rate": 0.00017770132758788103, + "loss": 0.3122232818603516, + "mean_token_accuracy": 0.8964972496032715, + "num_tokens": 2425068.0, + "step": 1700 + }, + { + "entropy": 0.35544642210006716, + "epoch": 4.499356499356499, + "grad_norm": 0.6574566960334778, + "learning_rate": 0.00017234090835450445, + "loss": 0.3042313766479492, + "mean_token_accuracy": 0.8966764956712723, + "num_tokens": 2498321.0, + "step": 1750 + }, + { + "entropy": 0.35768423914909364, + "epoch": 4.628056628056628, + "grad_norm": 0.6159058809280396, + "learning_rate": 0.00016689184347576657, + "loss": 0.30206684112548826, + "mean_token_accuracy": 0.898514216542244, + "num_tokens": 2570321.0, + "step": 1800 + }, + { + "entropy": 0.37273951262235644, + "epoch": 4.756756756756757, + "grad_norm": 0.5503262281417847, + "learning_rate": 0.00016136510037621443, + "loss": 0.31558765411376954, + "mean_token_accuracy": 0.8939771527051925, + "num_tokens": 2642851.0, + "step": 1850 + }, + { + "entropy": 0.3542595499753952, + "epoch": 4.885456885456885, + "grad_norm": 0.7454004883766174, + "learning_rate": 0.0001557718028246568, + "loss": 0.3020258712768555, + "mean_token_accuracy": 0.8977061492204667, + "num_tokens": 2715629.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3610583918610799, + "eval_loss": 0.7293093204498291, + "eval_mean_token_accuracy": 0.8165675112881612, + "eval_num_tokens": 2777465.0, + "eval_runtime": 160.2029, + "eval_samples_per_second": 9.65, + "eval_steps_per_second": 1.211, + "step": 1945 + }, + { + "entropy": 0.3579569302424036, + "epoch": 5.012870012870013, + "grad_norm": 0.4158821403980255, + "learning_rate": 0.00015012320854516744, + "loss": 0.3004540252685547, + "mean_token_accuracy": 0.8991928124668622, + "num_tokens": 2784896.0, + "step": 1950 + }, + { + "entropy": 0.26410215690732003, + "epoch": 5.141570141570142, + "grad_norm": 0.5636745095252991, + "learning_rate": 0.00014443068655847486, + "loss": 0.20726581573486327, + "mean_token_accuracy": 0.9306374716758729, + "num_tokens": 2854374.0, + "step": 2000 + }, + { + "entropy": 0.26181382283568383, + "epoch": 5.27027027027027, + "grad_norm": 0.5340386033058167, + "learning_rate": 0.00013870569429934264, + "loss": 0.2055559730529785, + "mean_token_accuracy": 0.9296587175130844, + "num_tokens": 2927362.0, + "step": 2050 + }, + { + "entropy": 0.261084942817688, + "epoch": 5.398970398970399, + "grad_norm": 0.6604517102241516, + "learning_rate": 0.00013295975455599807, + "loss": 0.20649850845336915, + "mean_token_accuracy": 0.9292166239023208, + "num_tokens": 3000143.0, + "step": 2100 + }, + { + "entropy": 0.2605279567837715, + "epoch": 5.527670527670527, + "grad_norm": 0.6845375895500183, + "learning_rate": 0.00012720443227802214, + "loss": 0.20826065063476562, + "mean_token_accuracy": 0.9290810519456864, + "num_tokens": 3072561.0, + "step": 2150 + }, + { + "entropy": 0.2661572727560997, + "epoch": 5.656370656370656, + "grad_norm": 0.557294487953186, + "learning_rate": 0.00012145131129938128, + "loss": 0.2132364273071289, + "mean_token_accuracy": 0.9271187698841095, + "num_tokens": 3144831.0, + "step": 2200 + }, + { + "entropy": 0.2664845323562622, + "epoch": 5.785070785070785, + "grad_norm": 0.5024778842926025, + "learning_rate": 0.00011571197102344961, + "loss": 0.21299808502197265, + "mean_token_accuracy": 0.9263874608278274, + "num_tokens": 3215960.0, + "step": 2250 + }, + { + "entropy": 0.26299254536628724, + "epoch": 5.913770913770914, + "grad_norm": 0.634398341178894, + "learning_rate": 0.00010999796311694913, + "loss": 0.20765880584716798, + "mean_token_accuracy": 0.928708678483963, + "num_tokens": 3287499.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.26795706152915955, + "eval_loss": 0.8664970993995667, + "eval_mean_token_accuracy": 0.8188820490517568, + "eval_num_tokens": 3332958.0, + "eval_runtime": 160.1105, + "eval_samples_per_second": 9.656, + "eval_steps_per_second": 1.212, + "step": 2334 + }, + { + "entropy": 0.2456816165435194, + "epoch": 6.041184041184041, + "grad_norm": 0.638988196849823, + "learning_rate": 0.00010432078825971574, + "loss": 0.18737899780273437, + "mean_token_accuracy": 0.9367745241733513, + "num_tokens": 3356879.0, + "step": 2350 + }, + { + "entropy": 0.19167753487825392, + "epoch": 6.1698841698841695, + "grad_norm": 0.6846708059310913, + "learning_rate": 9.869187299708746e-05, + "loss": 0.1415392589569092, + "mean_token_accuracy": 0.9528085774183274, + "num_tokens": 3427587.0, + "step": 2400 + }, + { + "entropy": 0.19594975739717482, + "epoch": 6.298584298584299, + "grad_norm": 0.45718589425086975, + "learning_rate": 9.312254674150445e-05, + "loss": 0.14171995162963869, + "mean_token_accuracy": 0.9512906008958817, + "num_tokens": 3498688.0, + "step": 2450 + }, + { + "entropy": 0.20213496513664722, + "epoch": 6.427284427284428, + "grad_norm": 0.9218644499778748, + "learning_rate": 8.762401896960985e-05, + "loss": 0.14789142608642578, + "mean_token_accuracy": 0.9499747276306152, + "num_tokens": 3566137.0, + "step": 2500 + }, + { + "entropy": 0.18978311866521835, + "epoch": 6.555984555984556, + "grad_norm": 0.3909820318222046, + "learning_rate": 8.22073566607478e-05, + "loss": 0.1412898349761963, + "mean_token_accuracy": 0.951915112733841, + "num_tokens": 3639573.0, + "step": 2550 + }, + { + "entropy": 0.19396651685237884, + "epoch": 6.684684684684685, + "grad_norm": 0.633430540561676, + "learning_rate": 7.688346202226782e-05, + "loss": 0.14628902435302735, + "mean_token_accuracy": 0.9509830921888351, + "num_tokens": 3710433.0, + "step": 2600 + }, + { + "entropy": 0.19417933478951455, + "epoch": 6.813384813384813, + "grad_norm": 0.4060302972793579, + "learning_rate": 7.166305054646867e-05, + "loss": 0.14396389961242675, + "mean_token_accuracy": 0.9514253032207489, + "num_tokens": 3781550.0, + "step": 2650 + }, + { + "entropy": 0.19293704330921174, + "epoch": 6.942084942084942, + "grad_norm": 0.40285640954971313, + "learning_rate": 6.655662944334679e-05, + "loss": 0.14185072898864745, + "mean_token_accuracy": 0.951703981757164, + "num_tokens": 3854952.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.20662364801487973, + "eval_loss": 1.0061800479888916, + "eval_mean_token_accuracy": 0.8215526999886503, + "eval_num_tokens": 3888451.0, + "eval_runtime": 159.9122, + "eval_samples_per_second": 9.668, + "eval_steps_per_second": 1.213, + "step": 2723 + }, + { + "entropy": 0.17519116890851896, + "epoch": 7.06949806949807, + "grad_norm": 0.3706398904323578, + "learning_rate": 6.157447649255791e-05, + "loss": 0.12077541351318359, + "mean_token_accuracy": 0.9585421494763307, + "num_tokens": 3928354.0, + "step": 2750 + }, + { + "entropy": 0.16396746069192886, + "epoch": 7.198198198198198, + "grad_norm": 0.27986252307891846, + "learning_rate": 5.672661935715706e-05, + "loss": 0.11088459014892578, + "mean_token_accuracy": 0.9617730963230133, + "num_tokens": 4000113.0, + "step": 2800 + }, + { + "entropy": 0.16335458047688006, + "epoch": 7.326898326898327, + "grad_norm": 0.2767539322376251, + "learning_rate": 5.2022815400752626e-05, + "loss": 0.11108455657958985, + "mean_token_accuracy": 0.9625812923908234, + "num_tokens": 4071032.0, + "step": 2850 + }, + { + "entropy": 0.16325364008545876, + "epoch": 7.455598455598455, + "grad_norm": 0.42529481649398804, + "learning_rate": 4.74725320486962e-05, + "loss": 0.11113139152526856, + "mean_token_accuracy": 0.9612818968296051, + "num_tokens": 4143616.0, + "step": 2900 + }, + { + "entropy": 0.1575803980231285, + "epoch": 7.584298584298584, + "grad_norm": 0.2682833969593048, + "learning_rate": 4.308492773283644e-05, + "loss": 0.11050393104553223, + "mean_token_accuracy": 0.9619130200147629, + "num_tokens": 4215664.0, + "step": 2950 + }, + { + "entropy": 0.1647344709187746, + "epoch": 7.712998712998713, + "grad_norm": 0.44769325852394104, + "learning_rate": 3.8868833458189334e-05, + "loss": 0.11467294692993164, + "mean_token_accuracy": 0.9609260809421539, + "num_tokens": 4286387.0, + "step": 3000 + }, + { + "entropy": 0.16482041254639626, + "epoch": 7.841698841698841, + "grad_norm": 0.3471318781375885, + "learning_rate": 3.4832735028625626e-05, + "loss": 0.11508123397827148, + "mean_token_accuracy": 0.9613727742433548, + "num_tokens": 4355875.0, + "step": 3050 + }, + { + "entropy": 0.16096619956195354, + "epoch": 7.97039897039897, + "grad_norm": 0.5056831240653992, + "learning_rate": 3.098475596735076e-05, + "loss": 0.11199919700622558, + "mean_token_accuracy": 0.9618956363201141, + "num_tokens": 4427088.0, + "step": 3100 + }, + { + "epoch": 8.0, + "eval_entropy": 0.16944569894640715, + "eval_loss": 1.193907618522644, + "eval_mean_token_accuracy": 0.8212007554536013, + "eval_num_tokens": 4443944.0, + "eval_runtime": 159.9592, + "eval_samples_per_second": 9.665, + "eval_steps_per_second": 1.213, + "step": 3112 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.445001770940518e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..547215a6e90313f8b9e939d213ef8247cc319764 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.06684998388944888, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..382cae7e1ff49cf46b28b86b4db79d05c8646c57 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/trainer_state.json @@ -0,0 +1,833 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3501, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6304629689455032, + "epoch": 0.1287001287001287, + "grad_norm": 0.9736679196357727, + "learning_rate": 3.2321893373855725e-05, + "loss": 1.5586164855957032, + "mean_token_accuracy": 0.6625205805897713, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.865804158449173, + "epoch": 0.2574002574002574, + "grad_norm": 0.8218768239021301, + "learning_rate": 6.530341722472892e-05, + "loss": 0.7942887115478515, + "mean_token_accuracy": 0.7780012512207031, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7885943764448166, + "epoch": 0.3861003861003861, + "grad_norm": 0.6200866103172302, + "learning_rate": 9.828494107560211e-05, + "loss": 0.7239870452880859, + "mean_token_accuracy": 0.7923985701799393, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7731903672218323, + "epoch": 0.5148005148005148, + "grad_norm": 0.645903468132019, + "learning_rate": 0.00013126646492647528, + "loss": 0.7023859405517578, + "mean_token_accuracy": 0.7972224324941635, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7606095188856125, + "epoch": 0.6435006435006435, + "grad_norm": 0.6395116448402405, + "learning_rate": 0.00016424798877734848, + "loss": 0.6980245208740234, + "mean_token_accuracy": 0.8005052077770233, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.7547655135393143, + "epoch": 0.7722007722007722, + "grad_norm": 0.6048902869224548, + "learning_rate": 0.00019722951262822168, + "loss": 0.6896456146240234, + "mean_token_accuracy": 0.8002784997224808, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.750541335940361, + "epoch": 0.9009009009009009, + "grad_norm": 0.5950207114219666, + "learning_rate": 0.00023021103647909486, + "loss": 0.6888568878173829, + "mean_token_accuracy": 0.8017277073860168, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6049834216378399, + "eval_loss": 0.6702247858047485, + "eval_mean_token_accuracy": 0.8023447219243983, + "eval_num_tokens": 555493.0, + "eval_runtime": 160.7595, + "eval_samples_per_second": 9.617, + "eval_steps_per_second": 1.207, + "step": 389 + }, + { + "entropy": 0.7378247476587392, + "epoch": 1.0283140283140284, + "grad_norm": 0.7072293758392334, + "learning_rate": 0.0002565910901719884, + "loss": 0.6729380798339843, + "mean_token_accuracy": 0.8071240547931555, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7119287192821503, + "epoch": 1.157014157014157, + "grad_norm": 0.8078219294548035, + "learning_rate": 0.00025641034526723873, + "loss": 0.6499625396728516, + "mean_token_accuracy": 0.8119793623685837, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7278847670555115, + "epoch": 1.2857142857142856, + "grad_norm": 0.6917271018028259, + "learning_rate": 0.0002559717467472163, + "loss": 0.6674861145019532, + "mean_token_accuracy": 0.8067185115814209, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7150312864780426, + "epoch": 1.4144144144144144, + "grad_norm": 0.8094608187675476, + "learning_rate": 0.0002552761773865406, + "loss": 0.6532902526855469, + "mean_token_accuracy": 0.8110801106691361, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6913726192712784, + "epoch": 1.5431145431145432, + "grad_norm": 0.6806851625442505, + "learning_rate": 0.0002543250371693649, + "loss": 0.6355234527587891, + "mean_token_accuracy": 0.8154943329095841, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.716510910987854, + "epoch": 1.6718146718146718, + "grad_norm": 0.8052911162376404, + "learning_rate": 0.00025312024047160467, + "loss": 0.6559846496582031, + "mean_token_accuracy": 0.8098146516084671, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6818760082125663, + "epoch": 1.8005148005148004, + "grad_norm": 0.5995563268661499, + "learning_rate": 0.0002516642122078408, + "loss": 0.6270184326171875, + "mean_token_accuracy": 0.8169869929552078, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6728256559371948, + "epoch": 1.9292149292149292, + "grad_norm": 0.684234082698822, + "learning_rate": 0.00024995988295065347, + "loss": 0.622582130432129, + "mean_token_accuracy": 0.8187594223022461, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5827852641184306, + "eval_loss": 0.6126551628112793, + "eval_mean_token_accuracy": 0.8199613155777922, + "eval_num_tokens": 1110986.0, + "eval_runtime": 160.3137, + "eval_samples_per_second": 9.644, + "eval_steps_per_second": 1.21, + "step": 778 + }, + { + "entropy": 0.6285555227236315, + "epoch": 2.056628056628057, + "grad_norm": 0.729573130607605, + "learning_rate": 0.00024801068303220925, + "loss": 0.5709608840942383, + "mean_token_accuracy": 0.8302884445045934, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5837715390324593, + "epoch": 2.1853281853281854, + "grad_norm": 0.657953143119812, + "learning_rate": 0.00024582053563997405, + "loss": 0.5185433197021484, + "mean_token_accuracy": 0.8406957858800888, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.5731439217925072, + "epoch": 2.314028314028314, + "grad_norm": 0.7121743559837341, + "learning_rate": 0.00024339384892044734, + "loss": 0.5192452621459961, + "mean_token_accuracy": 0.8415917128324508, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5781797507405281, + "epoch": 2.4427284427284426, + "grad_norm": 0.583035409450531, + "learning_rate": 0.00024073550710681184, + "loss": 0.51892822265625, + "mean_token_accuracy": 0.8394862592220307, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5838576689362526, + "epoch": 2.571428571428571, + "grad_norm": 0.7375105619430542, + "learning_rate": 0.00023785086068835518, + "loss": 0.5248625946044921, + "mean_token_accuracy": 0.8392664766311646, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5801274874806404, + "epoch": 2.7001287001287, + "grad_norm": 0.5255026817321777, + "learning_rate": 0.00023474571564145, + "loss": 0.5241841125488281, + "mean_token_accuracy": 0.8405563312768937, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.582165597975254, + "epoch": 2.828828828828829, + "grad_norm": 0.6157832145690918, + "learning_rate": 0.00023142632174376774, + "loss": 0.5229883956909179, + "mean_token_accuracy": 0.8406760370731354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5778958490490913, + "epoch": 2.9575289575289574, + "grad_norm": 0.5259988307952881, + "learning_rate": 0.000227899359995246, + "loss": 0.5213600540161133, + "mean_token_accuracy": 0.8410600781440735, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5146706245609165, + "eval_loss": 0.6217455267906189, + "eval_mean_token_accuracy": 0.8225257052588708, + "eval_num_tokens": 1666479.0, + "eval_runtime": 159.8044, + "eval_samples_per_second": 9.674, + "eval_steps_per_second": 1.214, + "step": 1167 + }, + { + "entropy": 0.5129444800844096, + "epoch": 3.0849420849420848, + "grad_norm": 0.6443549394607544, + "learning_rate": 0.00022417192917112742, + "loss": 0.4500067520141602, + "mean_token_accuracy": 0.8588406985456293, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.45547543197870255, + "epoch": 3.213642213642214, + "grad_norm": 0.6051533818244934, + "learning_rate": 0.0002202515315341355, + "loss": 0.3920443344116211, + "mean_token_accuracy": 0.8705952340364456, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4667788457870483, + "epoch": 3.3423423423423424, + "grad_norm": 0.625999927520752, + "learning_rate": 0.0002161460577345441, + "loss": 0.4082052230834961, + "mean_token_accuracy": 0.8685764318704605, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.49247278362512587, + "epoch": 3.471042471042471, + "grad_norm": 0.7345328330993652, + "learning_rate": 0.000211863770928533, + "loss": 0.4317153549194336, + "mean_token_accuracy": 0.8622037881612777, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.48875897347927094, + "epoch": 3.5997425997425996, + "grad_norm": 0.6730979681015015, + "learning_rate": 0.00020741329014679433, + "loss": 0.4286677932739258, + "mean_token_accuracy": 0.8626715314388275, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.4849738264083862, + "epoch": 3.7284427284427286, + "grad_norm": 0.6888502240180969, + "learning_rate": 0.00020280357294686466, + "loss": 0.4235057830810547, + "mean_token_accuracy": 0.8637587743997573, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.4883710837364197, + "epoch": 3.857142857142857, + "grad_norm": 0.6455050110816956, + "learning_rate": 0.00019804389738409812, + "loss": 0.42961734771728516, + "mean_token_accuracy": 0.8626390981674195, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.46666738361120225, + "epoch": 3.985842985842986, + "grad_norm": 0.6892870664596558, + "learning_rate": 0.0001931438433375683, + "loss": 0.4121334457397461, + "mean_token_accuracy": 0.8682902973890304, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4674856377016638, + "eval_loss": 0.6489306688308716, + "eval_mean_token_accuracy": 0.8197588545759928, + "eval_num_tokens": 2221972.0, + "eval_runtime": 159.7918, + "eval_samples_per_second": 9.675, + "eval_steps_per_second": 1.214, + "step": 1556 + }, + { + "entropy": 0.3744586084527199, + "epoch": 4.113256113256114, + "grad_norm": 0.6487554311752319, + "learning_rate": 0.00018811327322848455, + "loss": 0.31006853103637694, + "mean_token_accuracy": 0.8973448577553335, + "num_tokens": 2282164.0, + "step": 1600 + }, + { + "entropy": 0.3579921764135361, + "epoch": 4.241956241956242, + "grad_norm": 0.6199011206626892, + "learning_rate": 0.00018296231216993105, + "loss": 0.2979485511779785, + "mean_token_accuracy": 0.8998763573169708, + "num_tokens": 2353213.0, + "step": 1650 + }, + { + "entropy": 0.3664910554885864, + "epoch": 4.370656370656371, + "grad_norm": 0.7049311399459839, + "learning_rate": 0.00017770132758788103, + "loss": 0.3122232818603516, + "mean_token_accuracy": 0.8964972496032715, + "num_tokens": 2425068.0, + "step": 1700 + }, + { + "entropy": 0.35544642210006716, + "epoch": 4.499356499356499, + "grad_norm": 0.6574566960334778, + "learning_rate": 0.00017234090835450445, + "loss": 0.3042313766479492, + "mean_token_accuracy": 0.8966764956712723, + "num_tokens": 2498321.0, + "step": 1750 + }, + { + "entropy": 0.35768423914909364, + "epoch": 4.628056628056628, + "grad_norm": 0.6159058809280396, + "learning_rate": 0.00016689184347576657, + "loss": 0.30206684112548826, + "mean_token_accuracy": 0.898514216542244, + "num_tokens": 2570321.0, + "step": 1800 + }, + { + "entropy": 0.37273951262235644, + "epoch": 4.756756756756757, + "grad_norm": 0.5503262281417847, + "learning_rate": 0.00016136510037621443, + "loss": 0.31558765411376954, + "mean_token_accuracy": 0.8939771527051925, + "num_tokens": 2642851.0, + "step": 1850 + }, + { + "entropy": 0.3542595499753952, + "epoch": 4.885456885456885, + "grad_norm": 0.7454004883766174, + "learning_rate": 0.0001557718028246568, + "loss": 0.3020258712768555, + "mean_token_accuracy": 0.8977061492204667, + "num_tokens": 2715629.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3610583918610799, + "eval_loss": 0.7293093204498291, + "eval_mean_token_accuracy": 0.8165675112881612, + "eval_num_tokens": 2777465.0, + "eval_runtime": 160.2029, + "eval_samples_per_second": 9.65, + "eval_steps_per_second": 1.211, + "step": 1945 + }, + { + "entropy": 0.3579569302424036, + "epoch": 5.012870012870013, + "grad_norm": 0.4158821403980255, + "learning_rate": 0.00015012320854516744, + "loss": 0.3004540252685547, + "mean_token_accuracy": 0.8991928124668622, + "num_tokens": 2784896.0, + "step": 1950 + }, + { + "entropy": 0.26410215690732003, + "epoch": 5.141570141570142, + "grad_norm": 0.5636745095252991, + "learning_rate": 0.00014443068655847486, + "loss": 0.20726581573486327, + "mean_token_accuracy": 0.9306374716758729, + "num_tokens": 2854374.0, + "step": 2000 + }, + { + "entropy": 0.26181382283568383, + "epoch": 5.27027027027027, + "grad_norm": 0.5340386033058167, + "learning_rate": 0.00013870569429934264, + "loss": 0.2055559730529785, + "mean_token_accuracy": 0.9296587175130844, + "num_tokens": 2927362.0, + "step": 2050 + }, + { + "entropy": 0.261084942817688, + "epoch": 5.398970398970399, + "grad_norm": 0.6604517102241516, + "learning_rate": 0.00013295975455599807, + "loss": 0.20649850845336915, + "mean_token_accuracy": 0.9292166239023208, + "num_tokens": 3000143.0, + "step": 2100 + }, + { + "entropy": 0.2605279567837715, + "epoch": 5.527670527670527, + "grad_norm": 0.6845375895500183, + "learning_rate": 0.00012720443227802214, + "loss": 0.20826065063476562, + "mean_token_accuracy": 0.9290810519456864, + "num_tokens": 3072561.0, + "step": 2150 + }, + { + "entropy": 0.2661572727560997, + "epoch": 5.656370656370656, + "grad_norm": 0.557294487953186, + "learning_rate": 0.00012145131129938128, + "loss": 0.2132364273071289, + "mean_token_accuracy": 0.9271187698841095, + "num_tokens": 3144831.0, + "step": 2200 + }, + { + "entropy": 0.2664845323562622, + "epoch": 5.785070785070785, + "grad_norm": 0.5024778842926025, + "learning_rate": 0.00011571197102344961, + "loss": 0.21299808502197265, + "mean_token_accuracy": 0.9263874608278274, + "num_tokens": 3215960.0, + "step": 2250 + }, + { + "entropy": 0.26299254536628724, + "epoch": 5.913770913770914, + "grad_norm": 0.634398341178894, + "learning_rate": 0.00010999796311694913, + "loss": 0.20765880584716798, + "mean_token_accuracy": 0.928708678483963, + "num_tokens": 3287499.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.26795706152915955, + "eval_loss": 0.8664970993995667, + "eval_mean_token_accuracy": 0.8188820490517568, + "eval_num_tokens": 3332958.0, + "eval_runtime": 160.1105, + "eval_samples_per_second": 9.656, + "eval_steps_per_second": 1.212, + "step": 2334 + }, + { + "entropy": 0.2456816165435194, + "epoch": 6.041184041184041, + "grad_norm": 0.638988196849823, + "learning_rate": 0.00010432078825971574, + "loss": 0.18737899780273437, + "mean_token_accuracy": 0.9367745241733513, + "num_tokens": 3356879.0, + "step": 2350 + }, + { + "entropy": 0.19167753487825392, + "epoch": 6.1698841698841695, + "grad_norm": 0.6846708059310913, + "learning_rate": 9.869187299708746e-05, + "loss": 0.1415392589569092, + "mean_token_accuracy": 0.9528085774183274, + "num_tokens": 3427587.0, + "step": 2400 + }, + { + "entropy": 0.19594975739717482, + "epoch": 6.298584298584299, + "grad_norm": 0.45718589425086975, + "learning_rate": 9.312254674150445e-05, + "loss": 0.14171995162963869, + "mean_token_accuracy": 0.9512906008958817, + "num_tokens": 3498688.0, + "step": 2450 + }, + { + "entropy": 0.20213496513664722, + "epoch": 6.427284427284428, + "grad_norm": 0.9218644499778748, + "learning_rate": 8.762401896960985e-05, + "loss": 0.14789142608642578, + "mean_token_accuracy": 0.9499747276306152, + "num_tokens": 3566137.0, + "step": 2500 + }, + { + "entropy": 0.18978311866521835, + "epoch": 6.555984555984556, + "grad_norm": 0.3909820318222046, + "learning_rate": 8.22073566607478e-05, + "loss": 0.1412898349761963, + "mean_token_accuracy": 0.951915112733841, + "num_tokens": 3639573.0, + "step": 2550 + }, + { + "entropy": 0.19396651685237884, + "epoch": 6.684684684684685, + "grad_norm": 0.633430540561676, + "learning_rate": 7.688346202226782e-05, + "loss": 0.14628902435302735, + "mean_token_accuracy": 0.9509830921888351, + "num_tokens": 3710433.0, + "step": 2600 + }, + { + "entropy": 0.19417933478951455, + "epoch": 6.813384813384813, + "grad_norm": 0.4060302972793579, + "learning_rate": 7.166305054646867e-05, + "loss": 0.14396389961242675, + "mean_token_accuracy": 0.9514253032207489, + "num_tokens": 3781550.0, + "step": 2650 + }, + { + "entropy": 0.19293704330921174, + "epoch": 6.942084942084942, + "grad_norm": 0.40285640954971313, + "learning_rate": 6.655662944334679e-05, + "loss": 0.14185072898864745, + "mean_token_accuracy": 0.951703981757164, + "num_tokens": 3854952.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.20662364801487973, + "eval_loss": 1.0061800479888916, + "eval_mean_token_accuracy": 0.8215526999886503, + "eval_num_tokens": 3888451.0, + "eval_runtime": 159.9122, + "eval_samples_per_second": 9.668, + "eval_steps_per_second": 1.213, + "step": 2723 + }, + { + "entropy": 0.17519116890851896, + "epoch": 7.06949806949807, + "grad_norm": 0.3706398904323578, + "learning_rate": 6.157447649255791e-05, + "loss": 0.12077541351318359, + "mean_token_accuracy": 0.9585421494763307, + "num_tokens": 3928354.0, + "step": 2750 + }, + { + "entropy": 0.16396746069192886, + "epoch": 7.198198198198198, + "grad_norm": 0.27986252307891846, + "learning_rate": 5.672661935715706e-05, + "loss": 0.11088459014892578, + "mean_token_accuracy": 0.9617730963230133, + "num_tokens": 4000113.0, + "step": 2800 + }, + { + "entropy": 0.16335458047688006, + "epoch": 7.326898326898327, + "grad_norm": 0.2767539322376251, + "learning_rate": 5.2022815400752626e-05, + "loss": 0.11108455657958985, + "mean_token_accuracy": 0.9625812923908234, + "num_tokens": 4071032.0, + "step": 2850 + }, + { + "entropy": 0.16325364008545876, + "epoch": 7.455598455598455, + "grad_norm": 0.42529481649398804, + "learning_rate": 4.74725320486962e-05, + "loss": 0.11113139152526856, + "mean_token_accuracy": 0.9612818968296051, + "num_tokens": 4143616.0, + "step": 2900 + }, + { + "entropy": 0.1575803980231285, + "epoch": 7.584298584298584, + "grad_norm": 0.2682833969593048, + "learning_rate": 4.308492773283644e-05, + "loss": 0.11050393104553223, + "mean_token_accuracy": 0.9619130200147629, + "num_tokens": 4215664.0, + "step": 2950 + }, + { + "entropy": 0.1647344709187746, + "epoch": 7.712998712998713, + "grad_norm": 0.44769325852394104, + "learning_rate": 3.8868833458189334e-05, + "loss": 0.11467294692993164, + "mean_token_accuracy": 0.9609260809421539, + "num_tokens": 4286387.0, + "step": 3000 + }, + { + "entropy": 0.16482041254639626, + "epoch": 7.841698841698841, + "grad_norm": 0.3471318781375885, + "learning_rate": 3.4832735028625626e-05, + "loss": 0.11508123397827148, + "mean_token_accuracy": 0.9613727742433548, + "num_tokens": 4355875.0, + "step": 3050 + }, + { + "entropy": 0.16096619956195354, + "epoch": 7.97039897039897, + "grad_norm": 0.5056831240653992, + "learning_rate": 3.098475596735076e-05, + "loss": 0.11199919700622558, + "mean_token_accuracy": 0.9618956363201141, + "num_tokens": 4427088.0, + "step": 3100 + }, + { + "epoch": 8.0, + "eval_entropy": 0.16944569894640715, + "eval_loss": 1.193907618522644, + "eval_mean_token_accuracy": 0.8212007554536013, + "eval_num_tokens": 4443944.0, + "eval_runtime": 159.9592, + "eval_samples_per_second": 9.665, + "eval_steps_per_second": 1.213, + "step": 3112 + }, + { + "entropy": 0.14968648966815737, + "epoch": 8.097812097812097, + "grad_norm": 0.16369523108005524, + "learning_rate": 2.733264116655299e-05, + "loss": 0.10023937225341797, + "mean_token_accuracy": 0.9659906402982846, + "num_tokens": 4497867.0, + "step": 3150 + }, + { + "entropy": 0.1457908333837986, + "epoch": 8.226512226512227, + "grad_norm": 0.34754154086112976, + "learning_rate": 2.388374129912877e-05, + "loss": 0.098594970703125, + "mean_token_accuracy": 0.9657697731256485, + "num_tokens": 4568534.0, + "step": 3200 + }, + { + "entropy": 0.14762556083500386, + "epoch": 8.355212355212355, + "grad_norm": 0.15934935212135315, + "learning_rate": 2.0644998023859455e-05, + "loss": 0.09606503486633301, + "mean_token_accuracy": 0.965254881978035, + "num_tokens": 4642530.0, + "step": 3250 + }, + { + "entropy": 0.15189546518027783, + "epoch": 8.483912483912484, + "grad_norm": 0.1770402044057846, + "learning_rate": 1.7622930013818186e-05, + "loss": 0.10185425758361816, + "mean_token_accuracy": 0.9637523639202118, + "num_tokens": 4711573.0, + "step": 3300 + }, + { + "entropy": 0.15004121020436287, + "epoch": 8.612612612612612, + "grad_norm": 0.16243170201778412, + "learning_rate": 1.4823619836126707e-05, + "loss": 0.10156736373901368, + "mean_token_accuracy": 0.964007465839386, + "num_tokens": 4782666.0, + "step": 3350 + }, + { + "entropy": 0.1525618066638708, + "epoch": 8.741312741312742, + "grad_norm": 0.184408500790596, + "learning_rate": 1.225270170947043e-05, + "loss": 0.10364385604858399, + "mean_token_accuracy": 0.9634579128026962, + "num_tokens": 4852180.0, + "step": 3400 + }, + { + "entropy": 0.1454480966180563, + "epoch": 8.87001287001287, + "grad_norm": 0.2857225239276886, + "learning_rate": 9.915350164011423e-06, + "loss": 0.09707194328308105, + "mean_token_accuracy": 0.965444205403328, + "num_tokens": 4926402.0, + "step": 3450 + }, + { + "entropy": 0.1478308391571045, + "epoch": 8.998712998712998, + "grad_norm": 0.1329645812511444, + "learning_rate": 7.816269626524821e-06, + "loss": 0.09891718864440918, + "mean_token_accuracy": 0.9649236488342285, + "num_tokens": 4998841.0, + "step": 3500 + }, + { + "epoch": 9.0, + "eval_entropy": 0.15559418468745714, + "eval_loss": 1.2841299772262573, + "eval_mean_token_accuracy": 0.8217662496050608, + "eval_num_tokens": 4999437.0, + "eval_runtime": 159.9591, + "eval_samples_per_second": 9.665, + "eval_steps_per_second": 1.213, + "step": 3501 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.377780321673011e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..547215a6e90313f8b9e939d213ef8247cc319764 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.06684998388944888, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e377e1bbf2edf85b7b8c37a1e2bf414db08a8dc2 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/trainer_state.json @@ -0,0 +1,115 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 389, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6304629689455032, + "epoch": 0.1287001287001287, + "grad_norm": 0.9736679196357727, + "learning_rate": 3.2321893373855725e-05, + "loss": 1.5586164855957032, + "mean_token_accuracy": 0.6625205805897713, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.865804158449173, + "epoch": 0.2574002574002574, + "grad_norm": 0.8218768239021301, + "learning_rate": 6.530341722472892e-05, + "loss": 0.7942887115478515, + "mean_token_accuracy": 0.7780012512207031, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7885943764448166, + "epoch": 0.3861003861003861, + "grad_norm": 0.6200866103172302, + "learning_rate": 9.828494107560211e-05, + "loss": 0.7239870452880859, + "mean_token_accuracy": 0.7923985701799393, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7731903672218323, + "epoch": 0.5148005148005148, + "grad_norm": 0.645903468132019, + "learning_rate": 0.00013126646492647528, + "loss": 0.7023859405517578, + "mean_token_accuracy": 0.7972224324941635, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7606095188856125, + "epoch": 0.6435006435006435, + "grad_norm": 0.6395116448402405, + "learning_rate": 0.00016424798877734848, + "loss": 0.6980245208740234, + "mean_token_accuracy": 0.8005052077770233, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.7547655135393143, + "epoch": 0.7722007722007722, + "grad_norm": 0.6048902869224548, + "learning_rate": 0.00019722951262822168, + "loss": 0.6896456146240234, + "mean_token_accuracy": 0.8002784997224808, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.750541335940361, + "epoch": 0.9009009009009009, + "grad_norm": 0.5950207114219666, + "learning_rate": 0.00023021103647909486, + "loss": 0.6888568878173829, + "mean_token_accuracy": 0.8017277073860168, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6049834216378399, + "eval_loss": 0.6702247858047485, + "eval_mean_token_accuracy": 0.8023447219243983, + "eval_num_tokens": 555493.0, + "eval_runtime": 160.7595, + "eval_samples_per_second": 9.617, + "eval_steps_per_second": 1.207, + "step": 389 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.306612280369152e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..547215a6e90313f8b9e939d213ef8247cc319764 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.06684998388944888, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d870187002cdc862eda4eb796b2bba2e3f42ac53 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/trainer_state.json @@ -0,0 +1,914 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 3890, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6304629689455032, + "epoch": 0.1287001287001287, + "grad_norm": 0.9736679196357727, + "learning_rate": 3.2321893373855725e-05, + "loss": 1.5586164855957032, + "mean_token_accuracy": 0.6625205805897713, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.865804158449173, + "epoch": 0.2574002574002574, + "grad_norm": 0.8218768239021301, + "learning_rate": 6.530341722472892e-05, + "loss": 0.7942887115478515, + "mean_token_accuracy": 0.7780012512207031, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7885943764448166, + "epoch": 0.3861003861003861, + "grad_norm": 0.6200866103172302, + "learning_rate": 9.828494107560211e-05, + "loss": 0.7239870452880859, + "mean_token_accuracy": 0.7923985701799393, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7731903672218323, + "epoch": 0.5148005148005148, + "grad_norm": 0.645903468132019, + "learning_rate": 0.00013126646492647528, + "loss": 0.7023859405517578, + "mean_token_accuracy": 0.7972224324941635, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7606095188856125, + "epoch": 0.6435006435006435, + "grad_norm": 0.6395116448402405, + "learning_rate": 0.00016424798877734848, + "loss": 0.6980245208740234, + "mean_token_accuracy": 0.8005052077770233, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.7547655135393143, + "epoch": 0.7722007722007722, + "grad_norm": 0.6048902869224548, + "learning_rate": 0.00019722951262822168, + "loss": 0.6896456146240234, + "mean_token_accuracy": 0.8002784997224808, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.750541335940361, + "epoch": 0.9009009009009009, + "grad_norm": 0.5950207114219666, + "learning_rate": 0.00023021103647909486, + "loss": 0.6888568878173829, + "mean_token_accuracy": 0.8017277073860168, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6049834216378399, + "eval_loss": 0.6702247858047485, + "eval_mean_token_accuracy": 0.8023447219243983, + "eval_num_tokens": 555493.0, + "eval_runtime": 160.7595, + "eval_samples_per_second": 9.617, + "eval_steps_per_second": 1.207, + "step": 389 + }, + { + "entropy": 0.7378247476587392, + "epoch": 1.0283140283140284, + "grad_norm": 0.7072293758392334, + "learning_rate": 0.0002565910901719884, + "loss": 0.6729380798339843, + "mean_token_accuracy": 0.8071240547931555, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7119287192821503, + "epoch": 1.157014157014157, + "grad_norm": 0.8078219294548035, + "learning_rate": 0.00025641034526723873, + "loss": 0.6499625396728516, + "mean_token_accuracy": 0.8119793623685837, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7278847670555115, + "epoch": 1.2857142857142856, + "grad_norm": 0.6917271018028259, + "learning_rate": 0.0002559717467472163, + "loss": 0.6674861145019532, + "mean_token_accuracy": 0.8067185115814209, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7150312864780426, + "epoch": 1.4144144144144144, + "grad_norm": 0.8094608187675476, + "learning_rate": 0.0002552761773865406, + "loss": 0.6532902526855469, + "mean_token_accuracy": 0.8110801106691361, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6913726192712784, + "epoch": 1.5431145431145432, + "grad_norm": 0.6806851625442505, + "learning_rate": 0.0002543250371693649, + "loss": 0.6355234527587891, + "mean_token_accuracy": 0.8154943329095841, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.716510910987854, + "epoch": 1.6718146718146718, + "grad_norm": 0.8052911162376404, + "learning_rate": 0.00025312024047160467, + "loss": 0.6559846496582031, + "mean_token_accuracy": 0.8098146516084671, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6818760082125663, + "epoch": 1.8005148005148004, + "grad_norm": 0.5995563268661499, + "learning_rate": 0.0002516642122078408, + "loss": 0.6270184326171875, + "mean_token_accuracy": 0.8169869929552078, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6728256559371948, + "epoch": 1.9292149292149292, + "grad_norm": 0.684234082698822, + "learning_rate": 0.00024995988295065347, + "loss": 0.622582130432129, + "mean_token_accuracy": 0.8187594223022461, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5827852641184306, + "eval_loss": 0.6126551628112793, + "eval_mean_token_accuracy": 0.8199613155777922, + "eval_num_tokens": 1110986.0, + "eval_runtime": 160.3137, + "eval_samples_per_second": 9.644, + "eval_steps_per_second": 1.21, + "step": 778 + }, + { + "entropy": 0.6285555227236315, + "epoch": 2.056628056628057, + "grad_norm": 0.729573130607605, + "learning_rate": 0.00024801068303220925, + "loss": 0.5709608840942383, + "mean_token_accuracy": 0.8302884445045934, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5837715390324593, + "epoch": 2.1853281853281854, + "grad_norm": 0.657953143119812, + "learning_rate": 0.00024582053563997405, + "loss": 0.5185433197021484, + "mean_token_accuracy": 0.8406957858800888, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.5731439217925072, + "epoch": 2.314028314028314, + "grad_norm": 0.7121743559837341, + "learning_rate": 0.00024339384892044734, + "loss": 0.5192452621459961, + "mean_token_accuracy": 0.8415917128324508, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5781797507405281, + "epoch": 2.4427284427284426, + "grad_norm": 0.583035409450531, + "learning_rate": 0.00024073550710681184, + "loss": 0.51892822265625, + "mean_token_accuracy": 0.8394862592220307, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5838576689362526, + "epoch": 2.571428571428571, + "grad_norm": 0.7375105619430542, + "learning_rate": 0.00023785086068835518, + "loss": 0.5248625946044921, + "mean_token_accuracy": 0.8392664766311646, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5801274874806404, + "epoch": 2.7001287001287, + "grad_norm": 0.5255026817321777, + "learning_rate": 0.00023474571564145, + "loss": 0.5241841125488281, + "mean_token_accuracy": 0.8405563312768937, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.582165597975254, + "epoch": 2.828828828828829, + "grad_norm": 0.6157832145690918, + "learning_rate": 0.00023142632174376774, + "loss": 0.5229883956909179, + "mean_token_accuracy": 0.8406760370731354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5778958490490913, + "epoch": 2.9575289575289574, + "grad_norm": 0.5259988307952881, + "learning_rate": 0.000227899359995246, + "loss": 0.5213600540161133, + "mean_token_accuracy": 0.8410600781440735, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5146706245609165, + "eval_loss": 0.6217455267906189, + "eval_mean_token_accuracy": 0.8225257052588708, + "eval_num_tokens": 1666479.0, + "eval_runtime": 159.8044, + "eval_samples_per_second": 9.674, + "eval_steps_per_second": 1.214, + "step": 1167 + }, + { + "entropy": 0.5129444800844096, + "epoch": 3.0849420849420848, + "grad_norm": 0.6443549394607544, + "learning_rate": 0.00022417192917112742, + "loss": 0.4500067520141602, + "mean_token_accuracy": 0.8588406985456293, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.45547543197870255, + "epoch": 3.213642213642214, + "grad_norm": 0.6051533818244934, + "learning_rate": 0.0002202515315341355, + "loss": 0.3920443344116211, + "mean_token_accuracy": 0.8705952340364456, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4667788457870483, + "epoch": 3.3423423423423424, + "grad_norm": 0.625999927520752, + "learning_rate": 0.0002161460577345441, + "loss": 0.4082052230834961, + "mean_token_accuracy": 0.8685764318704605, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.49247278362512587, + "epoch": 3.471042471042471, + "grad_norm": 0.7345328330993652, + "learning_rate": 0.000211863770928533, + "loss": 0.4317153549194336, + "mean_token_accuracy": 0.8622037881612777, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.48875897347927094, + "epoch": 3.5997425997425996, + "grad_norm": 0.6730979681015015, + "learning_rate": 0.00020741329014679433, + "loss": 0.4286677932739258, + "mean_token_accuracy": 0.8626715314388275, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.4849738264083862, + "epoch": 3.7284427284427286, + "grad_norm": 0.6888502240180969, + "learning_rate": 0.00020280357294686466, + "loss": 0.4235057830810547, + "mean_token_accuracy": 0.8637587743997573, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.4883710837364197, + "epoch": 3.857142857142857, + "grad_norm": 0.6455050110816956, + "learning_rate": 0.00019804389738409812, + "loss": 0.42961734771728516, + "mean_token_accuracy": 0.8626390981674195, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.46666738361120225, + "epoch": 3.985842985842986, + "grad_norm": 0.6892870664596558, + "learning_rate": 0.0001931438433375683, + "loss": 0.4121334457397461, + "mean_token_accuracy": 0.8682902973890304, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4674856377016638, + "eval_loss": 0.6489306688308716, + "eval_mean_token_accuracy": 0.8197588545759928, + "eval_num_tokens": 2221972.0, + "eval_runtime": 159.7918, + "eval_samples_per_second": 9.675, + "eval_steps_per_second": 1.214, + "step": 1556 + }, + { + "entropy": 0.3744586084527199, + "epoch": 4.113256113256114, + "grad_norm": 0.6487554311752319, + "learning_rate": 0.00018811327322848455, + "loss": 0.31006853103637694, + "mean_token_accuracy": 0.8973448577553335, + "num_tokens": 2282164.0, + "step": 1600 + }, + { + "entropy": 0.3579921764135361, + "epoch": 4.241956241956242, + "grad_norm": 0.6199011206626892, + "learning_rate": 0.00018296231216993105, + "loss": 0.2979485511779785, + "mean_token_accuracy": 0.8998763573169708, + "num_tokens": 2353213.0, + "step": 1650 + }, + { + "entropy": 0.3664910554885864, + "epoch": 4.370656370656371, + "grad_norm": 0.7049311399459839, + "learning_rate": 0.00017770132758788103, + "loss": 0.3122232818603516, + "mean_token_accuracy": 0.8964972496032715, + "num_tokens": 2425068.0, + "step": 1700 + }, + { + "entropy": 0.35544642210006716, + "epoch": 4.499356499356499, + "grad_norm": 0.6574566960334778, + "learning_rate": 0.00017234090835450445, + "loss": 0.3042313766479492, + "mean_token_accuracy": 0.8966764956712723, + "num_tokens": 2498321.0, + "step": 1750 + }, + { + "entropy": 0.35768423914909364, + "epoch": 4.628056628056628, + "grad_norm": 0.6159058809280396, + "learning_rate": 0.00016689184347576657, + "loss": 0.30206684112548826, + "mean_token_accuracy": 0.898514216542244, + "num_tokens": 2570321.0, + "step": 1800 + }, + { + "entropy": 0.37273951262235644, + "epoch": 4.756756756756757, + "grad_norm": 0.5503262281417847, + "learning_rate": 0.00016136510037621443, + "loss": 0.31558765411376954, + "mean_token_accuracy": 0.8939771527051925, + "num_tokens": 2642851.0, + "step": 1850 + }, + { + "entropy": 0.3542595499753952, + "epoch": 4.885456885456885, + "grad_norm": 0.7454004883766174, + "learning_rate": 0.0001557718028246568, + "loss": 0.3020258712768555, + "mean_token_accuracy": 0.8977061492204667, + "num_tokens": 2715629.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3610583918610799, + "eval_loss": 0.7293093204498291, + "eval_mean_token_accuracy": 0.8165675112881612, + "eval_num_tokens": 2777465.0, + "eval_runtime": 160.2029, + "eval_samples_per_second": 9.65, + "eval_steps_per_second": 1.211, + "step": 1945 + }, + { + "entropy": 0.3579569302424036, + "epoch": 5.012870012870013, + "grad_norm": 0.4158821403980255, + "learning_rate": 0.00015012320854516744, + "loss": 0.3004540252685547, + "mean_token_accuracy": 0.8991928124668622, + "num_tokens": 2784896.0, + "step": 1950 + }, + { + "entropy": 0.26410215690732003, + "epoch": 5.141570141570142, + "grad_norm": 0.5636745095252991, + "learning_rate": 0.00014443068655847486, + "loss": 0.20726581573486327, + "mean_token_accuracy": 0.9306374716758729, + "num_tokens": 2854374.0, + "step": 2000 + }, + { + "entropy": 0.26181382283568383, + "epoch": 5.27027027027027, + "grad_norm": 0.5340386033058167, + "learning_rate": 0.00013870569429934264, + "loss": 0.2055559730529785, + "mean_token_accuracy": 0.9296587175130844, + "num_tokens": 2927362.0, + "step": 2050 + }, + { + "entropy": 0.261084942817688, + "epoch": 5.398970398970399, + "grad_norm": 0.6604517102241516, + "learning_rate": 0.00013295975455599807, + "loss": 0.20649850845336915, + "mean_token_accuracy": 0.9292166239023208, + "num_tokens": 3000143.0, + "step": 2100 + }, + { + "entropy": 0.2605279567837715, + "epoch": 5.527670527670527, + "grad_norm": 0.6845375895500183, + "learning_rate": 0.00012720443227802214, + "loss": 0.20826065063476562, + "mean_token_accuracy": 0.9290810519456864, + "num_tokens": 3072561.0, + "step": 2150 + }, + { + "entropy": 0.2661572727560997, + "epoch": 5.656370656370656, + "grad_norm": 0.557294487953186, + "learning_rate": 0.00012145131129938128, + "loss": 0.2132364273071289, + "mean_token_accuracy": 0.9271187698841095, + "num_tokens": 3144831.0, + "step": 2200 + }, + { + "entropy": 0.2664845323562622, + "epoch": 5.785070785070785, + "grad_norm": 0.5024778842926025, + "learning_rate": 0.00011571197102344961, + "loss": 0.21299808502197265, + "mean_token_accuracy": 0.9263874608278274, + "num_tokens": 3215960.0, + "step": 2250 + }, + { + "entropy": 0.26299254536628724, + "epoch": 5.913770913770914, + "grad_norm": 0.634398341178894, + "learning_rate": 0.00010999796311694913, + "loss": 0.20765880584716798, + "mean_token_accuracy": 0.928708678483963, + "num_tokens": 3287499.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.26795706152915955, + "eval_loss": 0.8664970993995667, + "eval_mean_token_accuracy": 0.8188820490517568, + "eval_num_tokens": 3332958.0, + "eval_runtime": 160.1105, + "eval_samples_per_second": 9.656, + "eval_steps_per_second": 1.212, + "step": 2334 + }, + { + "entropy": 0.2456816165435194, + "epoch": 6.041184041184041, + "grad_norm": 0.638988196849823, + "learning_rate": 0.00010432078825971574, + "loss": 0.18737899780273437, + "mean_token_accuracy": 0.9367745241733513, + "num_tokens": 3356879.0, + "step": 2350 + }, + { + "entropy": 0.19167753487825392, + "epoch": 6.1698841698841695, + "grad_norm": 0.6846708059310913, + "learning_rate": 9.869187299708746e-05, + "loss": 0.1415392589569092, + "mean_token_accuracy": 0.9528085774183274, + "num_tokens": 3427587.0, + "step": 2400 + }, + { + "entropy": 0.19594975739717482, + "epoch": 6.298584298584299, + "grad_norm": 0.45718589425086975, + "learning_rate": 9.312254674150445e-05, + "loss": 0.14171995162963869, + "mean_token_accuracy": 0.9512906008958817, + "num_tokens": 3498688.0, + "step": 2450 + }, + { + "entropy": 0.20213496513664722, + "epoch": 6.427284427284428, + "grad_norm": 0.9218644499778748, + "learning_rate": 8.762401896960985e-05, + "loss": 0.14789142608642578, + "mean_token_accuracy": 0.9499747276306152, + "num_tokens": 3566137.0, + "step": 2500 + }, + { + "entropy": 0.18978311866521835, + "epoch": 6.555984555984556, + "grad_norm": 0.3909820318222046, + "learning_rate": 8.22073566607478e-05, + "loss": 0.1412898349761963, + "mean_token_accuracy": 0.951915112733841, + "num_tokens": 3639573.0, + "step": 2550 + }, + { + "entropy": 0.19396651685237884, + "epoch": 6.684684684684685, + "grad_norm": 0.633430540561676, + "learning_rate": 7.688346202226782e-05, + "loss": 0.14628902435302735, + "mean_token_accuracy": 0.9509830921888351, + "num_tokens": 3710433.0, + "step": 2600 + }, + { + "entropy": 0.19417933478951455, + "epoch": 6.813384813384813, + "grad_norm": 0.4060302972793579, + "learning_rate": 7.166305054646867e-05, + "loss": 0.14396389961242675, + "mean_token_accuracy": 0.9514253032207489, + "num_tokens": 3781550.0, + "step": 2650 + }, + { + "entropy": 0.19293704330921174, + "epoch": 6.942084942084942, + "grad_norm": 0.40285640954971313, + "learning_rate": 6.655662944334679e-05, + "loss": 0.14185072898864745, + "mean_token_accuracy": 0.951703981757164, + "num_tokens": 3854952.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.20662364801487973, + "eval_loss": 1.0061800479888916, + "eval_mean_token_accuracy": 0.8215526999886503, + "eval_num_tokens": 3888451.0, + "eval_runtime": 159.9122, + "eval_samples_per_second": 9.668, + "eval_steps_per_second": 1.213, + "step": 2723 + }, + { + "entropy": 0.17519116890851896, + "epoch": 7.06949806949807, + "grad_norm": 0.3706398904323578, + "learning_rate": 6.157447649255791e-05, + "loss": 0.12077541351318359, + "mean_token_accuracy": 0.9585421494763307, + "num_tokens": 3928354.0, + "step": 2750 + }, + { + "entropy": 0.16396746069192886, + "epoch": 7.198198198198198, + "grad_norm": 0.27986252307891846, + "learning_rate": 5.672661935715706e-05, + "loss": 0.11088459014892578, + "mean_token_accuracy": 0.9617730963230133, + "num_tokens": 4000113.0, + "step": 2800 + }, + { + "entropy": 0.16335458047688006, + "epoch": 7.326898326898327, + "grad_norm": 0.2767539322376251, + "learning_rate": 5.2022815400752626e-05, + "loss": 0.11108455657958985, + "mean_token_accuracy": 0.9625812923908234, + "num_tokens": 4071032.0, + "step": 2850 + }, + { + "entropy": 0.16325364008545876, + "epoch": 7.455598455598455, + "grad_norm": 0.42529481649398804, + "learning_rate": 4.74725320486962e-05, + "loss": 0.11113139152526856, + "mean_token_accuracy": 0.9612818968296051, + "num_tokens": 4143616.0, + "step": 2900 + }, + { + "entropy": 0.1575803980231285, + "epoch": 7.584298584298584, + "grad_norm": 0.2682833969593048, + "learning_rate": 4.308492773283644e-05, + "loss": 0.11050393104553223, + "mean_token_accuracy": 0.9619130200147629, + "num_tokens": 4215664.0, + "step": 2950 + }, + { + "entropy": 0.1647344709187746, + "epoch": 7.712998712998713, + "grad_norm": 0.44769325852394104, + "learning_rate": 3.8868833458189334e-05, + "loss": 0.11467294692993164, + "mean_token_accuracy": 0.9609260809421539, + "num_tokens": 4286387.0, + "step": 3000 + }, + { + "entropy": 0.16482041254639626, + "epoch": 7.841698841698841, + "grad_norm": 0.3471318781375885, + "learning_rate": 3.4832735028625626e-05, + "loss": 0.11508123397827148, + "mean_token_accuracy": 0.9613727742433548, + "num_tokens": 4355875.0, + "step": 3050 + }, + { + "entropy": 0.16096619956195354, + "epoch": 7.97039897039897, + "grad_norm": 0.5056831240653992, + "learning_rate": 3.098475596735076e-05, + "loss": 0.11199919700622558, + "mean_token_accuracy": 0.9618956363201141, + "num_tokens": 4427088.0, + "step": 3100 + }, + { + "epoch": 8.0, + "eval_entropy": 0.16944569894640715, + "eval_loss": 1.193907618522644, + "eval_mean_token_accuracy": 0.8212007554536013, + "eval_num_tokens": 4443944.0, + "eval_runtime": 159.9592, + "eval_samples_per_second": 9.665, + "eval_steps_per_second": 1.213, + "step": 3112 + }, + { + "entropy": 0.14968648966815737, + "epoch": 8.097812097812097, + "grad_norm": 0.16369523108005524, + "learning_rate": 2.733264116655299e-05, + "loss": 0.10023937225341797, + "mean_token_accuracy": 0.9659906402982846, + "num_tokens": 4497867.0, + "step": 3150 + }, + { + "entropy": 0.1457908333837986, + "epoch": 8.226512226512227, + "grad_norm": 0.34754154086112976, + "learning_rate": 2.388374129912877e-05, + "loss": 0.098594970703125, + "mean_token_accuracy": 0.9657697731256485, + "num_tokens": 4568534.0, + "step": 3200 + }, + { + "entropy": 0.14762556083500386, + "epoch": 8.355212355212355, + "grad_norm": 0.15934935212135315, + "learning_rate": 2.0644998023859455e-05, + "loss": 0.09606503486633301, + "mean_token_accuracy": 0.965254881978035, + "num_tokens": 4642530.0, + "step": 3250 + }, + { + "entropy": 0.15189546518027783, + "epoch": 8.483912483912484, + "grad_norm": 0.1770402044057846, + "learning_rate": 1.7622930013818186e-05, + "loss": 0.10185425758361816, + "mean_token_accuracy": 0.9637523639202118, + "num_tokens": 4711573.0, + "step": 3300 + }, + { + "entropy": 0.15004121020436287, + "epoch": 8.612612612612612, + "grad_norm": 0.16243170201778412, + "learning_rate": 1.4823619836126707e-05, + "loss": 0.10156736373901368, + "mean_token_accuracy": 0.964007465839386, + "num_tokens": 4782666.0, + "step": 3350 + }, + { + "entropy": 0.1525618066638708, + "epoch": 8.741312741312742, + "grad_norm": 0.184408500790596, + "learning_rate": 1.225270170947043e-05, + "loss": 0.10364385604858399, + "mean_token_accuracy": 0.9634579128026962, + "num_tokens": 4852180.0, + "step": 3400 + }, + { + "entropy": 0.1454480966180563, + "epoch": 8.87001287001287, + "grad_norm": 0.2857225239276886, + "learning_rate": 9.915350164011423e-06, + "loss": 0.09707194328308105, + "mean_token_accuracy": 0.965444205403328, + "num_tokens": 4926402.0, + "step": 3450 + }, + { + "entropy": 0.1478308391571045, + "epoch": 8.998712998712998, + "grad_norm": 0.1329645812511444, + "learning_rate": 7.816269626524821e-06, + "loss": 0.09891718864440918, + "mean_token_accuracy": 0.9649236488342285, + "num_tokens": 4998841.0, + "step": 3500 + }, + { + "epoch": 9.0, + "eval_entropy": 0.15559418468745714, + "eval_loss": 1.2841299772262573, + "eval_mean_token_accuracy": 0.8217662496050608, + "eval_num_tokens": 4999437.0, + "eval_runtime": 159.9591, + "eval_samples_per_second": 9.665, + "eval_steps_per_second": 1.213, + "step": 3501 + }, + { + "entropy": 0.14615772318358372, + "epoch": 9.126126126126126, + "grad_norm": 0.1925354301929474, + "learning_rate": 5.959684951719613e-06, + "loss": 0.09416162490844726, + "mean_token_accuracy": 0.9663437887875721, + "num_tokens": 5069076.0, + "step": 3550 + }, + { + "entropy": 0.14310922361910344, + "epoch": 9.254826254826256, + "grad_norm": 0.113509900867939, + "learning_rate": 4.3493329188027825e-06, + "loss": 0.09511731147766113, + "mean_token_accuracy": 0.9668506371974945, + "num_tokens": 5139230.0, + "step": 3600 + }, + { + "entropy": 0.1406601908057928, + "epoch": 9.383526383526384, + "grad_norm": 0.13180668652057648, + "learning_rate": 2.9884547104006827e-06, + "loss": 0.09364951133728028, + "mean_token_accuracy": 0.9671342217922211, + "num_tokens": 5210983.0, + "step": 3650 + }, + { + "entropy": 0.14347786396741868, + "epoch": 9.512226512226512, + "grad_norm": 0.14092309772968292, + "learning_rate": 1.8797893889764342e-06, + "loss": 0.09556793212890625, + "mean_token_accuracy": 0.9656666827201843, + "num_tokens": 5282246.0, + "step": 3700 + }, + { + "entropy": 0.1425378493219614, + "epoch": 9.64092664092664, + "grad_norm": 0.1836232841014862, + "learning_rate": 1.02556838387268e-06, + "loss": 0.09487992286682129, + "mean_token_accuracy": 0.9660949373245239, + "num_tokens": 5353348.0, + "step": 3750 + }, + { + "entropy": 0.14517559669911861, + "epoch": 9.76962676962677, + "grad_norm": 0.14669862389564514, + "learning_rate": 4.2751100007634374e-07, + "loss": 0.09604022026062012, + "mean_token_accuracy": 0.9655501979589463, + "num_tokens": 5423909.0, + "step": 3800 + }, + { + "entropy": 0.13744916424155235, + "epoch": 9.898326898326898, + "grad_norm": 0.1657056212425232, + "learning_rate": 8.682095774443924e-08, + "loss": 0.09153496742248535, + "mean_token_accuracy": 0.9672546869516373, + "num_tokens": 5498358.0, + "step": 3850 + }, + { + "epoch": 10.0, + "eval_entropy": 0.15181667365364193, + "eval_loss": 1.314748764038086, + "eval_mean_token_accuracy": 0.8213970894666062, + "eval_num_tokens": 5554930.0, + "eval_runtime": 160.0924, + "eval_samples_per_second": 9.657, + "eval_steps_per_second": 1.212, + "step": 3890 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 9.30762612374784e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..547215a6e90313f8b9e939d213ef8247cc319764 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.06684998388944888, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fa11e0fd870e95c46c274fc8bd93dc08087d50e5 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/trainer_state.json @@ -0,0 +1,206 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 778, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6304629689455032, + "epoch": 0.1287001287001287, + "grad_norm": 0.9736679196357727, + "learning_rate": 3.2321893373855725e-05, + "loss": 1.5586164855957032, + "mean_token_accuracy": 0.6625205805897713, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.865804158449173, + "epoch": 0.2574002574002574, + "grad_norm": 0.8218768239021301, + "learning_rate": 6.530341722472892e-05, + "loss": 0.7942887115478515, + "mean_token_accuracy": 0.7780012512207031, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7885943764448166, + "epoch": 0.3861003861003861, + "grad_norm": 0.6200866103172302, + "learning_rate": 9.828494107560211e-05, + "loss": 0.7239870452880859, + "mean_token_accuracy": 0.7923985701799393, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7731903672218323, + "epoch": 0.5148005148005148, + "grad_norm": 0.645903468132019, + "learning_rate": 0.00013126646492647528, + "loss": 0.7023859405517578, + "mean_token_accuracy": 0.7972224324941635, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7606095188856125, + "epoch": 0.6435006435006435, + "grad_norm": 0.6395116448402405, + "learning_rate": 0.00016424798877734848, + "loss": 0.6980245208740234, + "mean_token_accuracy": 0.8005052077770233, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.7547655135393143, + "epoch": 0.7722007722007722, + "grad_norm": 0.6048902869224548, + "learning_rate": 0.00019722951262822168, + "loss": 0.6896456146240234, + "mean_token_accuracy": 0.8002784997224808, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.750541335940361, + "epoch": 0.9009009009009009, + "grad_norm": 0.5950207114219666, + "learning_rate": 0.00023021103647909486, + "loss": 0.6888568878173829, + "mean_token_accuracy": 0.8017277073860168, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6049834216378399, + "eval_loss": 0.6702247858047485, + "eval_mean_token_accuracy": 0.8023447219243983, + "eval_num_tokens": 555493.0, + "eval_runtime": 160.7595, + "eval_samples_per_second": 9.617, + "eval_steps_per_second": 1.207, + "step": 389 + }, + { + "entropy": 0.7378247476587392, + "epoch": 1.0283140283140284, + "grad_norm": 0.7072293758392334, + "learning_rate": 0.0002565910901719884, + "loss": 0.6729380798339843, + "mean_token_accuracy": 0.8071240547931555, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7119287192821503, + "epoch": 1.157014157014157, + "grad_norm": 0.8078219294548035, + "learning_rate": 0.00025641034526723873, + "loss": 0.6499625396728516, + "mean_token_accuracy": 0.8119793623685837, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7278847670555115, + "epoch": 1.2857142857142856, + "grad_norm": 0.6917271018028259, + "learning_rate": 0.0002559717467472163, + "loss": 0.6674861145019532, + "mean_token_accuracy": 0.8067185115814209, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7150312864780426, + "epoch": 1.4144144144144144, + "grad_norm": 0.8094608187675476, + "learning_rate": 0.0002552761773865406, + "loss": 0.6532902526855469, + "mean_token_accuracy": 0.8110801106691361, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6913726192712784, + "epoch": 1.5431145431145432, + "grad_norm": 0.6806851625442505, + "learning_rate": 0.0002543250371693649, + "loss": 0.6355234527587891, + "mean_token_accuracy": 0.8154943329095841, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.716510910987854, + "epoch": 1.6718146718146718, + "grad_norm": 0.8052911162376404, + "learning_rate": 0.00025312024047160467, + "loss": 0.6559846496582031, + "mean_token_accuracy": 0.8098146516084671, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6818760082125663, + "epoch": 1.8005148005148004, + "grad_norm": 0.5995563268661499, + "learning_rate": 0.0002516642122078408, + "loss": 0.6270184326171875, + "mean_token_accuracy": 0.8169869929552078, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6728256559371948, + "epoch": 1.9292149292149292, + "grad_norm": 0.684234082698822, + "learning_rate": 0.00024995988295065347, + "loss": 0.622582130432129, + "mean_token_accuracy": 0.8187594223022461, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5827852641184306, + "eval_loss": 0.6126551628112793, + "eval_mean_token_accuracy": 0.8199613155777922, + "eval_num_tokens": 1110986.0, + "eval_runtime": 160.3137, + "eval_samples_per_second": 9.644, + "eval_steps_per_second": 1.21, + "step": 778 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.8586194538985472e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..dc7af43d0aa1b459aa2eea72151437a3b45c97a6 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/trainer_state.json @@ -0,0 +1,914 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 3890, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6110110306739807, + "epoch": 0.1287001287001287, + "grad_norm": 0.8611119389533997, + "learning_rate": 3.4130257962133866e-05, + "loss": 1.53956298828125, + "mean_token_accuracy": 0.6652342769503593, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.8662820833921433, + "epoch": 0.2574002574002574, + "grad_norm": 0.7405035495758057, + "learning_rate": 6.895705180104598e-05, + "loss": 0.7929539489746094, + "mean_token_accuracy": 0.7786347842216492, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7912819278240204, + "epoch": 0.3861003861003861, + "grad_norm": 0.6105485558509827, + "learning_rate": 0.00010378384563995809, + "loss": 0.7236511993408203, + "mean_token_accuracy": 0.7925467795133591, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7777709531784057, + "epoch": 0.5148005148005148, + "grad_norm": 0.5781793594360352, + "learning_rate": 0.0001386106394788702, + "loss": 0.7024919891357422, + "mean_token_accuracy": 0.7969876372814179, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7632609683275223, + "epoch": 0.6435006435006435, + "grad_norm": 0.6553444862365723, + "learning_rate": 0.00017343743331778232, + "loss": 0.7000718688964844, + "mean_token_accuracy": 0.7994742071628571, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.754165632724762, + "epoch": 0.7722007722007722, + "grad_norm": 0.634222149848938, + "learning_rate": 0.00020826422715669444, + "loss": 0.6909049987792969, + "mean_token_accuracy": 0.8016809666156769, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.7530711203813553, + "epoch": 0.9009009009009009, + "grad_norm": 0.7020156383514404, + "learning_rate": 0.00024309102099560653, + "loss": 0.69554931640625, + "mean_token_accuracy": 0.8002807641029358, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.5728044164242204, + "eval_loss": 0.6785851716995239, + "eval_mean_token_accuracy": 0.798433246993527, + "eval_num_tokens": 555493.0, + "eval_runtime": 162.4129, + "eval_samples_per_second": 9.513, + "eval_steps_per_second": 1.194, + "step": 389 + }, + { + "entropy": 0.7489313946829902, + "epoch": 1.0283140283140284, + "grad_norm": 0.7532815933227539, + "learning_rate": 0.0002709470016827303, + "loss": 0.6856581878662109, + "mean_token_accuracy": 0.8023027079273956, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7238409864902496, + "epoch": 1.157014157014157, + "grad_norm": 1.2016215324401855, + "learning_rate": 0.0002707561443541359, + "loss": 0.6699818420410156, + "mean_token_accuracy": 0.8070302194356919, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7388938587903976, + "epoch": 1.2857142857142856, + "grad_norm": 0.7279272079467773, + "learning_rate": 0.0002702930068622498, + "loss": 0.6728517150878907, + "mean_token_accuracy": 0.8049580943584442, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7284485149383545, + "epoch": 1.4144144144144144, + "grad_norm": 0.8563987016677856, + "learning_rate": 0.0002695585213716931, + "loss": 0.6657986450195312, + "mean_token_accuracy": 0.8085588800907135, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6960959500074386, + "epoch": 1.5431145431145432, + "grad_norm": 0.5145474672317505, + "learning_rate": 0.0002685541661937683, + "loss": 0.6358638763427734, + "mean_token_accuracy": 0.8131621342897415, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.7220414417982102, + "epoch": 1.6718146718146718, + "grad_norm": 0.9381059408187866, + "learning_rate": 0.00026728196281103746, + "loss": 0.6531407928466797, + "mean_token_accuracy": 0.811619822382927, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6955459499359131, + "epoch": 1.8005148005148004, + "grad_norm": 0.6115108728408813, + "learning_rate": 0.0002657444718086503, + "loss": 0.6269588088989257, + "mean_token_accuracy": 0.8155373805761337, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6929812705516816, + "epoch": 1.9292149292149292, + "grad_norm": 0.749489426612854, + "learning_rate": 0.0002639447877206115, + "loss": 0.629054069519043, + "mean_token_accuracy": 0.8183021235466004, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5833694102223387, + "eval_loss": 0.6328718662261963, + "eval_mean_token_accuracy": 0.8159044071571114, + "eval_num_tokens": 1110986.0, + "eval_runtime": 161.885, + "eval_samples_per_second": 9.544, + "eval_steps_per_second": 1.198, + "step": 778 + }, + { + "entropy": 0.647241060480927, + "epoch": 2.056628056628057, + "grad_norm": 0.7070767879486084, + "learning_rate": 0.00026188653280135975, + "loss": 0.5823922348022461, + "mean_token_accuracy": 0.8265365301960647, + "num_tokens": 1141195.0, + "step": 800 + }, + { + "entropy": 0.5995378407835961, + "epoch": 2.1853281853281854, + "grad_norm": 0.8090486526489258, + "learning_rate": 0.0002595738497351955, + "loss": 0.5325597763061524, + "mean_token_accuracy": 0.8369336777925491, + "num_tokens": 1210708.0, + "step": 850 + }, + { + "entropy": 0.6043145382404327, + "epoch": 2.314028314028314, + "grad_norm": 0.8279913067817688, + "learning_rate": 0.00025701139329823054, + "loss": 0.5414446258544922, + "mean_token_accuracy": 0.8361396533250809, + "num_tokens": 1283441.0, + "step": 900 + }, + { + "entropy": 0.5953224584460258, + "epoch": 2.4427284427284426, + "grad_norm": 0.6075023412704468, + "learning_rate": 0.00025420432098964183, + "loss": 0.536654167175293, + "mean_token_accuracy": 0.8340093129873276, + "num_tokens": 1356418.0, + "step": 950 + }, + { + "entropy": 0.5998479858040809, + "epoch": 2.571428571428571, + "grad_norm": 1.0311471223831177, + "learning_rate": 0.0002511582826510862, + "loss": 0.5372924423217773, + "mean_token_accuracy": 0.8366045409440994, + "num_tokens": 1427797.0, + "step": 1000 + }, + { + "entropy": 0.5980158120393753, + "epoch": 2.7001287001287, + "grad_norm": 0.5971426367759705, + "learning_rate": 0.0002478794090951689, + "loss": 0.5392885208129883, + "mean_token_accuracy": 0.8347727072238922, + "num_tokens": 1498082.0, + "step": 1050 + }, + { + "entropy": 0.5990680930018425, + "epoch": 2.828828828828829, + "grad_norm": 0.5662627220153809, + "learning_rate": 0.0002443742997658538, + "loss": 0.5360498428344727, + "mean_token_accuracy": 0.8371847170591354, + "num_tokens": 1568798.0, + "step": 1100 + }, + { + "entropy": 0.5957039377093315, + "epoch": 2.9575289575289574, + "grad_norm": 0.5043798685073853, + "learning_rate": 0.00024065000945565205, + "loss": 0.5342231369018555, + "mean_token_accuracy": 0.8380735236406326, + "num_tokens": 1643449.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5346978819861854, + "eval_loss": 0.6255015134811401, + "eval_mean_token_accuracy": 0.8202014476368108, + "eval_num_tokens": 1666479.0, + "eval_runtime": 161.6098, + "eval_samples_per_second": 9.56, + "eval_steps_per_second": 1.2, + "step": 1167 + }, + { + "entropy": 0.5212209137401196, + "epoch": 3.0849420849420848, + "grad_norm": 0.7095440626144409, + "learning_rate": 0.00023671403410632178, + "loss": 0.45311901092529294, + "mean_token_accuracy": 0.856362871449403, + "num_tokens": 1713536.0, + "step": 1200 + }, + { + "entropy": 0.4701593083143234, + "epoch": 3.213642213642214, + "grad_norm": 0.6408083438873291, + "learning_rate": 0.0002325742957216607, + "loss": 0.39916397094726563, + "mean_token_accuracy": 0.8698061722517013, + "num_tokens": 1785609.0, + "step": 1250 + }, + { + "entropy": 0.4766591975092888, + "epoch": 3.3423423423423424, + "grad_norm": 0.6415093541145325, + "learning_rate": 0.0002282391264227552, + "loss": 0.4116698455810547, + "mean_token_accuracy": 0.8679435575008392, + "num_tokens": 1858651.0, + "step": 1300 + }, + { + "entropy": 0.4937947469949722, + "epoch": 3.471042471042471, + "grad_norm": 0.6549825072288513, + "learning_rate": 0.00022371725167778054, + "loss": 0.4296376037597656, + "mean_token_accuracy": 0.8609357953071595, + "num_tokens": 1928692.0, + "step": 1350 + }, + { + "entropy": 0.4920153194665909, + "epoch": 3.5997425997425996, + "grad_norm": 0.6452126502990723, + "learning_rate": 0.00021901777274010406, + "loss": 0.4307489013671875, + "mean_token_accuracy": 0.8606827831268311, + "num_tokens": 1998668.0, + "step": 1400 + }, + { + "entropy": 0.490042342543602, + "epoch": 3.7284427284427286, + "grad_norm": 0.5727734565734863, + "learning_rate": 0.0002141501483300395, + "loss": 0.4295254135131836, + "mean_token_accuracy": 0.8616545403003693, + "num_tokens": 2072809.0, + "step": 1450 + }, + { + "entropy": 0.49857193052768706, + "epoch": 3.857142857142857, + "grad_norm": 0.7732954025268555, + "learning_rate": 0.00020912417559712133, + "loss": 0.4289303207397461, + "mean_token_accuracy": 0.8616443765163422, + "num_tokens": 2142475.0, + "step": 1500 + }, + { + "entropy": 0.4746784272789955, + "epoch": 3.985842985842986, + "grad_norm": 0.5791187882423401, + "learning_rate": 0.00020394997040121726, + "loss": 0.4180263900756836, + "mean_token_accuracy": 0.866080379486084, + "num_tokens": 2214044.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4793926059585257, + "eval_loss": 0.627627968788147, + "eval_mean_token_accuracy": 0.8233870095813397, + "eval_num_tokens": 2221972.0, + "eval_runtime": 162.0225, + "eval_samples_per_second": 9.536, + "eval_steps_per_second": 1.197, + "step": 1556 + }, + { + "entropy": 0.38195489000792454, + "epoch": 4.113256113256114, + "grad_norm": 0.6002617478370667, + "learning_rate": 0.0001986379469521669, + "loss": 0.30819049835205076, + "mean_token_accuracy": 0.8977848634575353, + "num_tokens": 2282164.0, + "step": 1600 + }, + { + "entropy": 0.3655787402391434, + "epoch": 4.241956241956242, + "grad_norm": 0.7100041508674622, + "learning_rate": 0.00019319879684892634, + "loss": 0.29959835052490236, + "mean_token_accuracy": 0.8991208010911942, + "num_tokens": 2353213.0, + "step": 1650 + }, + { + "entropy": 0.3821141055226326, + "epoch": 4.370656370656371, + "grad_norm": 0.5848307013511658, + "learning_rate": 0.00018764346756040715, + "loss": 0.313802490234375, + "mean_token_accuracy": 0.895167955160141, + "num_tokens": 2425068.0, + "step": 1700 + }, + { + "entropy": 0.37083797007799146, + "epoch": 4.499356499356499, + "grad_norm": 0.6447024941444397, + "learning_rate": 0.00018198314039132143, + "loss": 0.30583988189697264, + "mean_token_accuracy": 0.8961733293533325, + "num_tokens": 2498321.0, + "step": 1750 + }, + { + "entropy": 0.3791545969247818, + "epoch": 4.628056628056628, + "grad_norm": 0.6575382351875305, + "learning_rate": 0.00017622920797738184, + "loss": 0.3088031005859375, + "mean_token_accuracy": 0.8960050916671753, + "num_tokens": 2570321.0, + "step": 1800 + }, + { + "entropy": 0.3946831756830215, + "epoch": 4.756756756756757, + "grad_norm": 0.5351552963256836, + "learning_rate": 0.00017039325135515207, + "loss": 0.3229162979125977, + "mean_token_accuracy": 0.8920552498102188, + "num_tokens": 2642851.0, + "step": 1850 + }, + { + "entropy": 0.37298239797353744, + "epoch": 4.885456885456885, + "grad_norm": 0.7624587416648865, + "learning_rate": 0.00016448701665269964, + "loss": 0.3067934799194336, + "mean_token_accuracy": 0.8951873427629471, + "num_tokens": 2715629.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3738150204887095, + "eval_loss": 0.7131896615028381, + "eval_mean_token_accuracy": 0.8207752468045225, + "eval_num_tokens": 2777465.0, + "eval_runtime": 162.1244, + "eval_samples_per_second": 9.53, + "eval_steps_per_second": 1.197, + "step": 1945 + }, + { + "entropy": 0.377820266617669, + "epoch": 5.012870012870013, + "grad_norm": 0.41990190744400024, + "learning_rate": 0.00015852239144796624, + "loss": 0.3058685111999512, + "mean_token_accuracy": 0.8964343480389527, + "num_tokens": 2784896.0, + "step": 1950 + }, + { + "entropy": 0.2714502356946468, + "epoch": 5.141570141570142, + "grad_norm": 0.422568678855896, + "learning_rate": 0.00015251138084243995, + "loss": 0.2093442153930664, + "mean_token_accuracy": 0.9311346983909607, + "num_tokens": 2854374.0, + "step": 2000 + }, + { + "entropy": 0.268475965410471, + "epoch": 5.27027027027027, + "grad_norm": 0.6637414693832397, + "learning_rate": 0.0001464660832982852, + "loss": 0.20736080169677734, + "mean_token_accuracy": 0.9289199805259705, + "num_tokens": 2927362.0, + "step": 2050 + }, + { + "entropy": 0.2644876340031624, + "epoch": 5.398970398970399, + "grad_norm": 0.47317707538604736, + "learning_rate": 0.00014039866628756467, + "loss": 0.20464908599853515, + "mean_token_accuracy": 0.9300856202840805, + "num_tokens": 3000143.0, + "step": 2100 + }, + { + "entropy": 0.2675253136456013, + "epoch": 5.527670527670527, + "grad_norm": 0.5253982543945312, + "learning_rate": 0.00013432134180256338, + "loss": 0.21154335021972656, + "mean_token_accuracy": 0.9283734840154648, + "num_tokens": 3072561.0, + "step": 2150 + }, + { + "entropy": 0.27213907435536383, + "epoch": 5.656370656370656, + "grad_norm": 0.46738553047180176, + "learning_rate": 0.00012824634177650664, + "loss": 0.21339216232299804, + "mean_token_accuracy": 0.9272083270549775, + "num_tokens": 3144831.0, + "step": 2200 + }, + { + "entropy": 0.2785488124191761, + "epoch": 5.785070785070785, + "grad_norm": 0.4469502866268158, + "learning_rate": 0.00012218589346414205, + "loss": 0.21601097106933595, + "mean_token_accuracy": 0.9255663657188415, + "num_tokens": 3215960.0, + "step": 2250 + }, + { + "entropy": 0.2699935150146484, + "epoch": 5.913770913770914, + "grad_norm": 0.7359778881072998, + "learning_rate": 0.00011615219483173828, + "loss": 0.20725584030151367, + "mean_token_accuracy": 0.9286630594730377, + "num_tokens": 3287499.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.26417383682174783, + "eval_loss": 0.8880229592323303, + "eval_mean_token_accuracy": 0.8159987201395723, + "eval_num_tokens": 3332958.0, + "eval_runtime": 162.0991, + "eval_samples_per_second": 9.531, + "eval_steps_per_second": 1.197, + "step": 2334 + }, + { + "entropy": 0.24814540704693458, + "epoch": 6.041184041184041, + "grad_norm": 0.4953760802745819, + "learning_rate": 0.00011015739000603316, + "loss": 0.18749794006347656, + "mean_token_accuracy": 0.9370789509831052, + "num_tokens": 3356879.0, + "step": 2350 + }, + { + "entropy": 0.19976271741092205, + "epoch": 6.1698841698841695, + "grad_norm": 0.4834803342819214, + "learning_rate": 0.00010421354483154553, + "loss": 0.14283526420593262, + "mean_token_accuracy": 0.9521516615152359, + "num_tokens": 3427587.0, + "step": 2400 + }, + { + "entropy": 0.2060488449037075, + "epoch": 6.298584298584299, + "grad_norm": 0.4888673722743988, + "learning_rate": 9.8332622585447e-05, + "loss": 0.14414511680603026, + "mean_token_accuracy": 0.9510996866226197, + "num_tokens": 3498688.0, + "step": 2450 + }, + { + "entropy": 0.2059111550450325, + "epoch": 6.427284427284428, + "grad_norm": 0.4064404368400574, + "learning_rate": 9.252645989887253e-05, + "loss": 0.14820143699645996, + "mean_token_accuracy": 0.9507584601640702, + "num_tokens": 3566137.0, + "step": 2500 + }, + { + "entropy": 0.19700154662132263, + "epoch": 6.555984555984556, + "grad_norm": 0.467965304851532, + "learning_rate": 8.680674293313417e-05, + "loss": 0.14303470611572267, + "mean_token_accuracy": 0.9515972435474396, + "num_tokens": 3639573.0, + "step": 2550 + }, + { + "entropy": 0.20180423602461814, + "epoch": 6.684684684684685, + "grad_norm": 0.36836138367652893, + "learning_rate": 8.118498385878736e-05, + "loss": 0.14280882835388184, + "mean_token_accuracy": 0.9515993863344192, + "num_tokens": 3710433.0, + "step": 2600 + }, + { + "entropy": 0.20024395987391472, + "epoch": 6.813384813384813, + "grad_norm": 0.38375866413116455, + "learning_rate": 7.567249768489171e-05, + "loss": 0.1427844524383545, + "mean_token_accuracy": 0.9524166631698608, + "num_tokens": 3781550.0, + "step": 2650 + }, + { + "entropy": 0.19561587080359458, + "epoch": 6.942084942084942, + "grad_norm": 0.41185441613197327, + "learning_rate": 7.028037948510187e-05, + "loss": 0.13993803024291993, + "mean_token_accuracy": 0.9522478264570237, + "num_tokens": 3854952.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.19501976062034823, + "eval_loss": 1.0653952360153198, + "eval_mean_token_accuracy": 0.8205490803595671, + "eval_num_tokens": 3888451.0, + "eval_runtime": 161.8533, + "eval_samples_per_second": 9.546, + "eval_steps_per_second": 1.199, + "step": 2723 + }, + { + "entropy": 0.17789882526855277, + "epoch": 7.06949806949807, + "grad_norm": 0.41413992643356323, + "learning_rate": 6.50194820664261e-05, + "loss": 0.12078390121459961, + "mean_token_accuracy": 0.9589925727458916, + "num_tokens": 3928354.0, + "step": 2750 + }, + { + "entropy": 0.16781829454004765, + "epoch": 7.198198198198198, + "grad_norm": 0.25806066393852234, + "learning_rate": 5.990039412559906e-05, + "loss": 0.10963023185729981, + "mean_token_accuracy": 0.9617267113924026, + "num_tokens": 4000113.0, + "step": 2800 + }, + { + "entropy": 0.1649068508297205, + "epoch": 7.326898326898327, + "grad_norm": 0.27411890029907227, + "learning_rate": 5.493341893703393e-05, + "loss": 0.11152458190917969, + "mean_token_accuracy": 0.9620639663934708, + "num_tokens": 4071032.0, + "step": 2850 + }, + { + "entropy": 0.161333369910717, + "epoch": 7.455598455598455, + "grad_norm": 0.24944494664669037, + "learning_rate": 5.0128553615248396e-05, + "loss": 0.1094522476196289, + "mean_token_accuracy": 0.962428919672966, + "num_tokens": 4143616.0, + "step": 2900 + }, + { + "entropy": 0.15613057143986225, + "epoch": 7.584298584298584, + "grad_norm": 0.1455036848783493, + "learning_rate": 4.549546899350423e-05, + "loss": 0.11092090606689453, + "mean_token_accuracy": 0.9620462411642074, + "num_tokens": 4215664.0, + "step": 2950 + }, + { + "entropy": 0.1631234459578991, + "epoch": 7.712998712998713, + "grad_norm": 0.2129560261964798, + "learning_rate": 4.104349015915862e-05, + "loss": 0.1141857624053955, + "mean_token_accuracy": 0.9613765001296997, + "num_tokens": 4286387.0, + "step": 3000 + }, + { + "entropy": 0.1680422095954418, + "epoch": 7.841698841698841, + "grad_norm": 0.24886097013950348, + "learning_rate": 3.678157768490372e-05, + "loss": 0.11513191223144531, + "mean_token_accuracy": 0.9615794748067856, + "num_tokens": 4355875.0, + "step": 3050 + }, + { + "entropy": 0.16354035697877406, + "epoch": 7.97039897039897, + "grad_norm": 0.27600204944610596, + "learning_rate": 3.27183095936714e-05, + "loss": 0.1118631362915039, + "mean_token_accuracy": 0.9623224419355393, + "num_tokens": 4427088.0, + "step": 3100 + }, + { + "epoch": 8.0, + "eval_entropy": 0.16586771200305409, + "eval_loss": 1.204746961593628, + "eval_mean_token_accuracy": 0.8229975042883882, + "eval_num_tokens": 4443944.0, + "eval_runtime": 162.0251, + "eval_samples_per_second": 9.536, + "eval_steps_per_second": 1.197, + "step": 3112 + }, + { + "entropy": 0.1515902608934075, + "epoch": 8.097812097812097, + "grad_norm": 0.14122211933135986, + "learning_rate": 2.88618640935022e-05, + "loss": 0.09900871276855469, + "mean_token_accuracy": 0.9665110737386376, + "num_tokens": 4497867.0, + "step": 3150 + }, + { + "entropy": 0.14593622356653213, + "epoch": 8.226512226512227, + "grad_norm": 0.20527532696723938, + "learning_rate": 2.5220003117128462e-05, + "loss": 0.09842084884643555, + "mean_token_accuracy": 0.9655911487340927, + "num_tokens": 4568534.0, + "step": 3200 + }, + { + "entropy": 0.1482392605394125, + "epoch": 8.355212355212355, + "grad_norm": 0.13207173347473145, + "learning_rate": 2.1800056699401584e-05, + "loss": 0.09551989555358886, + "mean_token_accuracy": 0.9650829958915711, + "num_tokens": 4642530.0, + "step": 3250 + }, + { + "entropy": 0.15284131653606892, + "epoch": 8.483912483912484, + "grad_norm": 0.1777282953262329, + "learning_rate": 1.860890822400777e-05, + "loss": 0.10169261932373047, + "mean_token_accuracy": 0.9635212075710297, + "num_tokens": 4711573.0, + "step": 3300 + }, + { + "entropy": 0.15095721945166587, + "epoch": 8.612612612612612, + "grad_norm": 0.14988408982753754, + "learning_rate": 1.5652980569165692e-05, + "loss": 0.10045011520385742, + "mean_token_accuracy": 0.96439110994339, + "num_tokens": 4782666.0, + "step": 3350 + }, + { + "entropy": 0.15411154814064504, + "epoch": 8.741312741312742, + "grad_norm": 0.14055995643138885, + "learning_rate": 1.2938223180191691e-05, + "loss": 0.1034860897064209, + "mean_token_accuracy": 0.963447842001915, + "num_tokens": 4852180.0, + "step": 3400 + }, + { + "entropy": 0.14739766091108322, + "epoch": 8.87001287001287, + "grad_norm": 0.15041407942771912, + "learning_rate": 1.0470100094950792e-05, + "loss": 0.09690508842468262, + "mean_token_accuracy": 0.96561603307724, + "num_tokens": 4926402.0, + "step": 3450 + }, + { + "entropy": 0.1498453303426504, + "epoch": 8.998712998712998, + "grad_norm": 0.1293368935585022, + "learning_rate": 8.253578946296125e-06, + "loss": 0.09874271392822266, + "mean_token_accuracy": 0.9647125631570816, + "num_tokens": 4998841.0, + "step": 3500 + }, + { + "epoch": 9.0, + "eval_entropy": 0.151532097729211, + "eval_loss": 1.302620768547058, + "eval_mean_token_accuracy": 0.8230925079473516, + "eval_num_tokens": 4999437.0, + "eval_runtime": 161.5816, + "eval_samples_per_second": 9.562, + "eval_steps_per_second": 1.201, + "step": 3501 + }, + { + "entropy": 0.14752683314410123, + "epoch": 9.126126126126126, + "grad_norm": 0.16032852232456207, + "learning_rate": 6.293120963630391e-06, + "loss": 0.09370619773864747, + "mean_token_accuracy": 0.9661390173314798, + "num_tokens": 5069076.0, + "step": 3550 + }, + { + "entropy": 0.14519502766430378, + "epoch": 9.254826254826256, + "grad_norm": 0.1136530265212059, + "learning_rate": 4.59267199371469e-06, + "loss": 0.09496000289916992, + "mean_token_accuracy": 0.9666073507070542, + "num_tokens": 5139230.0, + "step": 3600 + }, + { + "entropy": 0.14283505395054819, + "epoch": 9.383526383526384, + "grad_norm": 0.1119781956076622, + "learning_rate": 3.1556545587961024e-06, + "loss": 0.09344432830810546, + "mean_token_accuracy": 0.9670081204175949, + "num_tokens": 5210983.0, + "step": 3650 + }, + { + "entropy": 0.14515610218048094, + "epoch": 9.512226512226512, + "grad_norm": 0.12781645357608795, + "learning_rate": 1.984960968039795e-06, + "loss": 0.09529141426086425, + "mean_token_accuracy": 0.9657131034135819, + "num_tokens": 5282246.0, + "step": 3700 + }, + { + "entropy": 0.1440463538467884, + "epoch": 9.64092664092664, + "grad_norm": 0.16869987547397614, + "learning_rate": 1.0829474961295484e-06, + "loss": 0.09463245391845704, + "mean_token_accuracy": 0.9662202161550522, + "num_tokens": 5353348.0, + "step": 3750 + }, + { + "entropy": 0.14663653157651424, + "epoch": 9.76962676962677, + "grad_norm": 0.14733418822288513, + "learning_rate": 4.51429640754206e-07, + "loss": 0.09587678909301758, + "mean_token_accuracy": 0.9654789489507675, + "num_tokens": 5423909.0, + "step": 3800 + }, + { + "entropy": 0.13829420879483223, + "epoch": 9.898326898326898, + "grad_norm": 0.15077215433120728, + "learning_rate": 9.167846852480809e-08, + "loss": 0.09117506027221679, + "mean_token_accuracy": 0.9676015496253967, + "num_tokens": 5498358.0, + "step": 3850 + }, + { + "epoch": 10.0, + "eval_entropy": 0.1479573895021812, + "eval_loss": 1.3304139375686646, + "eval_mean_token_accuracy": 0.8228786372646844, + "eval_num_tokens": 5554930.0, + "eval_runtime": 161.0933, + "eval_samples_per_second": 9.591, + "eval_steps_per_second": 1.204, + "step": 3890 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 9.30762612374784e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/README.md b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/adapter_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cd0d97a8c611bcad7d4624b6232e1767e61fe6d3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.00237968804112545, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "o_proj", + "v_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/chat_template.jinja b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/tokenizer_config.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/trainer_state.json b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ec843847a45e4c368b1b320587e796db7871525d --- /dev/null +++ b/productivity_original_Estonian/Qwen3-14B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/trainer_state.json @@ -0,0 +1,206 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 778, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6110110306739807, + "epoch": 0.1287001287001287, + "grad_norm": 0.8611119389533997, + "learning_rate": 3.4130257962133866e-05, + "loss": 1.53956298828125, + "mean_token_accuracy": 0.6652342769503593, + "num_tokens": 73407.0, + "step": 50 + }, + { + "entropy": 0.8662820833921433, + "epoch": 0.2574002574002574, + "grad_norm": 0.7405035495758057, + "learning_rate": 6.895705180104598e-05, + "loss": 0.7929539489746094, + "mean_token_accuracy": 0.7786347842216492, + "num_tokens": 143994.0, + "step": 100 + }, + { + "entropy": 0.7912819278240204, + "epoch": 0.3861003861003861, + "grad_norm": 0.6105485558509827, + "learning_rate": 0.00010378384563995809, + "loss": 0.7236511993408203, + "mean_token_accuracy": 0.7925467795133591, + "num_tokens": 216171.0, + "step": 150 + }, + { + "entropy": 0.7777709531784057, + "epoch": 0.5148005148005148, + "grad_norm": 0.5781793594360352, + "learning_rate": 0.0001386106394788702, + "loss": 0.7024919891357422, + "mean_token_accuracy": 0.7969876372814179, + "num_tokens": 284702.0, + "step": 200 + }, + { + "entropy": 0.7632609683275223, + "epoch": 0.6435006435006435, + "grad_norm": 0.6553444862365723, + "learning_rate": 0.00017343743331778232, + "loss": 0.7000718688964844, + "mean_token_accuracy": 0.7994742071628571, + "num_tokens": 356393.0, + "step": 250 + }, + { + "entropy": 0.754165632724762, + "epoch": 0.7722007722007722, + "grad_norm": 0.634222149848938, + "learning_rate": 0.00020826422715669444, + "loss": 0.6909049987792969, + "mean_token_accuracy": 0.8016809666156769, + "num_tokens": 426916.0, + "step": 300 + }, + { + "entropy": 0.7530711203813553, + "epoch": 0.9009009009009009, + "grad_norm": 0.7020156383514404, + "learning_rate": 0.00024309102099560653, + "loss": 0.69554931640625, + "mean_token_accuracy": 0.8002807641029358, + "num_tokens": 499603.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.5728044164242204, + "eval_loss": 0.6785851716995239, + "eval_mean_token_accuracy": 0.798433246993527, + "eval_num_tokens": 555493.0, + "eval_runtime": 162.4129, + "eval_samples_per_second": 9.513, + "eval_steps_per_second": 1.194, + "step": 389 + }, + { + "entropy": 0.7489313946829902, + "epoch": 1.0283140283140284, + "grad_norm": 0.7532815933227539, + "learning_rate": 0.0002709470016827303, + "loss": 0.6856581878662109, + "mean_token_accuracy": 0.8023027079273956, + "num_tokens": 570813.0, + "step": 400 + }, + { + "entropy": 0.7238409864902496, + "epoch": 1.157014157014157, + "grad_norm": 1.2016215324401855, + "learning_rate": 0.0002707561443541359, + "loss": 0.6699818420410156, + "mean_token_accuracy": 0.8070302194356919, + "num_tokens": 642956.0, + "step": 450 + }, + { + "entropy": 0.7388938587903976, + "epoch": 1.2857142857142856, + "grad_norm": 0.7279272079467773, + "learning_rate": 0.0002702930068622498, + "loss": 0.6728517150878907, + "mean_token_accuracy": 0.8049580943584442, + "num_tokens": 714498.0, + "step": 500 + }, + { + "entropy": 0.7284485149383545, + "epoch": 1.4144144144144144, + "grad_norm": 0.8563987016677856, + "learning_rate": 0.0002695585213716931, + "loss": 0.6657986450195312, + "mean_token_accuracy": 0.8085588800907135, + "num_tokens": 785595.0, + "step": 550 + }, + { + "entropy": 0.6960959500074386, + "epoch": 1.5431145431145432, + "grad_norm": 0.5145474672317505, + "learning_rate": 0.0002685541661937683, + "loss": 0.6358638763427734, + "mean_token_accuracy": 0.8131621342897415, + "num_tokens": 857551.0, + "step": 600 + }, + { + "entropy": 0.7220414417982102, + "epoch": 1.6718146718146718, + "grad_norm": 0.9381059408187866, + "learning_rate": 0.00026728196281103746, + "loss": 0.6531407928466797, + "mean_token_accuracy": 0.811619822382927, + "num_tokens": 926864.0, + "step": 650 + }, + { + "entropy": 0.6955459499359131, + "epoch": 1.8005148005148004, + "grad_norm": 0.6115108728408813, + "learning_rate": 0.0002657444718086503, + "loss": 0.6269588088989257, + "mean_token_accuracy": 0.8155373805761337, + "num_tokens": 998824.0, + "step": 700 + }, + { + "entropy": 0.6929812705516816, + "epoch": 1.9292149292149292, + "grad_norm": 0.749489426612854, + "learning_rate": 0.0002639447877206115, + "loss": 0.629054069519043, + "mean_token_accuracy": 0.8183021235466004, + "num_tokens": 1069332.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5833694102223387, + "eval_loss": 0.6328718662261963, + "eval_mean_token_accuracy": 0.8159044071571114, + "eval_num_tokens": 1110986.0, + "eval_runtime": 161.885, + "eval_samples_per_second": 9.544, + "eval_steps_per_second": 1.198, + "step": 778 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.8586194538985472e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}