diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test1/README.md b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bff82c08416144a6a775be03d092db2ae011625a --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: transformers +model_name: Qwen3-14B-Base_original_features_structural_train_original_features_structural_test1 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3-14B-Base_original_features_structural_train_original_features_structural_test1 + +This model is a fine-tuned version of [Qwen/Qwen3-14B-Base](https://huggingface.co/Qwen/Qwen3-14B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/rutorgw2) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/README.md b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ab8bc305f4c2aae36a9ab9b82b106c15e2ff7165 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: transformers +model_name: Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2 + +This model is a fine-tuned version of [Qwen/Qwen3-14B-Base](https://huggingface.co/Qwen/Qwen3-14B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/hj2m59ad) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/README.md b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/adapter_config.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..368b2eaa099a70c44e0aa69e3cf7163ac54c47fa --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.030195310029781444, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "v_proj", + "down_proj", + "up_proj", + "k_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/chat_template.jinja b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/tokenizer_config.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/trainer_state.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..490de9157cda39ecb1b352b5dac66d08a4c52098 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/trainer_state.json @@ -0,0 +1,317 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1284, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6439054441452026, + "epoch": 0.11682242990654206, + "grad_norm": 0.49546942114830017, + "learning_rate": 1.7676515353317202e-05, + "loss": 1.6080712890625, + "mean_token_accuracy": 0.6552277088165284, + "num_tokens": 162248.0, + "step": 50 + }, + { + "entropy": 0.8895833045244217, + "epoch": 0.2336448598130841, + "grad_norm": 0.6087712645530701, + "learning_rate": 3.571377591792659e-05, + "loss": 0.8471942138671875, + "mean_token_accuracy": 0.7710993754863739, + "num_tokens": 327614.0, + "step": 100 + }, + { + "entropy": 0.7270394039154052, + "epoch": 0.35046728971962615, + "grad_norm": 0.4936729371547699, + "learning_rate": 5.375103648253598e-05, + "loss": 0.6940426635742187, + "mean_token_accuracy": 0.7985459506511688, + "num_tokens": 494582.0, + "step": 150 + }, + { + "entropy": 0.6873268783092499, + "epoch": 0.4672897196261682, + "grad_norm": 0.5290274620056152, + "learning_rate": 7.178829704714537e-05, + "loss": 0.6487248229980469, + "mean_token_accuracy": 0.8118354797363281, + "num_tokens": 653981.0, + "step": 200 + }, + { + "entropy": 0.6600921255350113, + "epoch": 0.5841121495327103, + "grad_norm": 0.3852632939815521, + "learning_rate": 8.982555761175476e-05, + "loss": 0.6241029357910156, + "mean_token_accuracy": 0.8171858608722686, + "num_tokens": 814840.0, + "step": 250 + }, + { + "entropy": 0.6286565268039703, + "epoch": 0.7009345794392523, + "grad_norm": 0.44116389751434326, + "learning_rate": 0.00010786281817636416, + "loss": 0.5951404190063476, + "mean_token_accuracy": 0.8241004174947739, + "num_tokens": 979542.0, + "step": 300 + }, + { + "entropy": 0.6199766182899475, + "epoch": 0.8177570093457944, + "grad_norm": 0.38574621081352234, + "learning_rate": 0.00012590007874097354, + "loss": 0.5904315948486328, + "mean_token_accuracy": 0.8250177747011185, + "num_tokens": 1147400.0, + "step": 350 + }, + { + "entropy": 0.5950348073244095, + "epoch": 0.9345794392523364, + "grad_norm": 0.41016682982444763, + "learning_rate": 0.0001439373393055829, + "loss": 0.5652639389038085, + "mean_token_accuracy": 0.83076107442379, + "num_tokens": 1310490.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7618740917026222, + "eval_loss": 0.6691045761108398, + "eval_mean_token_accuracy": 0.8110752314716191, + "eval_num_tokens": 1399670.0, + "eval_runtime": 45.674, + "eval_samples_per_second": 26.952, + "eval_steps_per_second": 3.372, + "step": 428 + }, + { + "entropy": 0.5888169056177139, + "epoch": 1.0514018691588785, + "grad_norm": 0.339750200510025, + "learning_rate": 0.00015438762800645052, + "loss": 0.5587311935424805, + "mean_token_accuracy": 0.832102472782135, + "num_tokens": 1471726.0, + "step": 450 + }, + { + "entropy": 0.5644756412506103, + "epoch": 1.1682242990654206, + "grad_norm": 0.32897934317588806, + "learning_rate": 0.00015426955859477223, + "loss": 0.5355723571777343, + "mean_token_accuracy": 0.8378035259246827, + "num_tokens": 1638060.0, + "step": 500 + }, + { + "entropy": 0.5603723293542862, + "epoch": 1.2850467289719627, + "grad_norm": 0.3474414646625519, + "learning_rate": 0.00015402334678635756, + "loss": 0.5328482437133789, + "mean_token_accuracy": 0.8384119784832, + "num_tokens": 1802005.0, + "step": 550 + }, + { + "entropy": 0.551156692802906, + "epoch": 1.4018691588785046, + "grad_norm": 0.21533823013305664, + "learning_rate": 0.00015364940195109232, + "loss": 0.5259516525268555, + "mean_token_accuracy": 0.8401002615690232, + "num_tokens": 1969267.0, + "step": 600 + }, + { + "entropy": 0.5456148293614388, + "epoch": 1.5186915887850467, + "grad_norm": 0.3005809485912323, + "learning_rate": 0.0001531483458372002, + "loss": 0.5163665008544922, + "mean_token_accuracy": 0.8423654067516327, + "num_tokens": 2134793.0, + "step": 650 + }, + { + "entropy": 0.540266016125679, + "epoch": 1.6355140186915889, + "grad_norm": 0.2895597815513611, + "learning_rate": 0.00015252101153747827, + "loss": 0.5159417343139648, + "mean_token_accuracy": 0.8422912752628327, + "num_tokens": 2300672.0, + "step": 700 + }, + { + "entropy": 0.5450576081871986, + "epoch": 1.7523364485981308, + "grad_norm": 0.26751118898391724, + "learning_rate": 0.00015176844210413562, + "loss": 0.5150370788574219, + "mean_token_accuracy": 0.8433581227064133, + "num_tokens": 2459315.0, + "step": 750 + }, + { + "entropy": 0.540363709628582, + "epoch": 1.8691588785046729, + "grad_norm": 0.25945547223091125, + "learning_rate": 0.0001508918888145379, + "loss": 0.5133833312988281, + "mean_token_accuracy": 0.8442667639255523, + "num_tokens": 2616878.0, + "step": 800 + }, + { + "entropy": 0.5406239181756973, + "epoch": 1.985981308411215, + "grad_norm": 0.2768362760543823, + "learning_rate": 0.0001498928090907414, + "loss": 0.5116788864135742, + "mean_token_accuracy": 0.8440770989656449, + "num_tokens": 2778909.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6602114913138476, + "eval_loss": 0.6344706416130066, + "eval_mean_token_accuracy": 0.8167721358212557, + "eval_num_tokens": 2799340.0, + "eval_runtime": 45.6608, + "eval_samples_per_second": 26.96, + "eval_steps_per_second": 3.373, + "step": 856 + }, + { + "entropy": 0.47262389928102494, + "epoch": 2.102803738317757, + "grad_norm": 0.2535194158554077, + "learning_rate": 0.00014877286407627584, + "loss": 0.4428855514526367, + "mean_token_accuracy": 0.8600562286376953, + "num_tokens": 2951115.0, + "step": 900 + }, + { + "entropy": 0.47976606965065005, + "epoch": 2.2196261682242993, + "grad_norm": 0.30369797348976135, + "learning_rate": 0.00014753391587420475, + "loss": 0.44907299041748044, + "mean_token_accuracy": 0.8582676541805268, + "num_tokens": 3115016.0, + "step": 950 + }, + { + "entropy": 0.47275764375925067, + "epoch": 2.336448598130841, + "grad_norm": 0.3171652853488922, + "learning_rate": 0.0001461780244510558, + "loss": 0.44435993194580076, + "mean_token_accuracy": 0.8601345640420913, + "num_tokens": 3274313.0, + "step": 1000 + }, + { + "entropy": 0.4735608258843422, + "epoch": 2.453271028037383, + "grad_norm": 0.29764947295188904, + "learning_rate": 0.00014470744421176885, + "loss": 0.44539962768554686, + "mean_token_accuracy": 0.8607082539796829, + "num_tokens": 3440379.0, + "step": 1050 + }, + { + "entropy": 0.4744651636481285, + "epoch": 2.5700934579439254, + "grad_norm": 0.23777751624584198, + "learning_rate": 0.00014312462025135595, + "loss": 0.44422054290771484, + "mean_token_accuracy": 0.859566843509674, + "num_tokens": 3606755.0, + "step": 1100 + }, + { + "entropy": 0.47177853137254716, + "epoch": 2.6869158878504673, + "grad_norm": 0.28442806005477905, + "learning_rate": 0.00014143218428950668, + "loss": 0.4431512069702148, + "mean_token_accuracy": 0.8603039968013764, + "num_tokens": 3766555.0, + "step": 1150 + }, + { + "entropy": 0.4803256571292877, + "epoch": 2.803738317757009, + "grad_norm": 0.2728305757045746, + "learning_rate": 0.00013963295029489697, + "loss": 0.44966583251953124, + "mean_token_accuracy": 0.8587929052114487, + "num_tokens": 3927164.0, + "step": 1200 + }, + { + "entropy": 0.4743427449464798, + "epoch": 2.9205607476635516, + "grad_norm": 0.29242947697639465, + "learning_rate": 0.0001377299098064774, + "loss": 0.4463462829589844, + "mean_token_accuracy": 0.8596956437826156, + "num_tokens": 4090727.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.611924365356371, + "eval_loss": 0.6364901661872864, + "eval_mean_token_accuracy": 0.8110434765165503, + "eval_num_tokens": 4199010.0, + "eval_runtime": 45.6331, + "eval_samples_per_second": 26.976, + "eval_steps_per_second": 3.375, + "step": 1284 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.7771258761728e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/README.md b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/adapter_config.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..368b2eaa099a70c44e0aa69e3cf7163ac54c47fa --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.030195310029781444, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "v_proj", + "down_proj", + "up_proj", + "k_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/chat_template.jinja b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/tokenizer_config.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/trainer_state.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..76c39d76e8df6413f1c336533b9b9394b92edcf1 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/trainer_state.json @@ -0,0 +1,418 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1712, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6439054441452026, + "epoch": 0.11682242990654206, + "grad_norm": 0.49546942114830017, + "learning_rate": 1.7676515353317202e-05, + "loss": 1.6080712890625, + "mean_token_accuracy": 0.6552277088165284, + "num_tokens": 162248.0, + "step": 50 + }, + { + "entropy": 0.8895833045244217, + "epoch": 0.2336448598130841, + "grad_norm": 0.6087712645530701, + "learning_rate": 3.571377591792659e-05, + "loss": 0.8471942138671875, + "mean_token_accuracy": 0.7710993754863739, + "num_tokens": 327614.0, + "step": 100 + }, + { + "entropy": 0.7270394039154052, + "epoch": 0.35046728971962615, + "grad_norm": 0.4936729371547699, + "learning_rate": 5.375103648253598e-05, + "loss": 0.6940426635742187, + "mean_token_accuracy": 0.7985459506511688, + "num_tokens": 494582.0, + "step": 150 + }, + { + "entropy": 0.6873268783092499, + "epoch": 0.4672897196261682, + "grad_norm": 0.5290274620056152, + "learning_rate": 7.178829704714537e-05, + "loss": 0.6487248229980469, + "mean_token_accuracy": 0.8118354797363281, + "num_tokens": 653981.0, + "step": 200 + }, + { + "entropy": 0.6600921255350113, + "epoch": 0.5841121495327103, + "grad_norm": 0.3852632939815521, + "learning_rate": 8.982555761175476e-05, + "loss": 0.6241029357910156, + "mean_token_accuracy": 0.8171858608722686, + "num_tokens": 814840.0, + "step": 250 + }, + { + "entropy": 0.6286565268039703, + "epoch": 0.7009345794392523, + "grad_norm": 0.44116389751434326, + "learning_rate": 0.00010786281817636416, + "loss": 0.5951404190063476, + "mean_token_accuracy": 0.8241004174947739, + "num_tokens": 979542.0, + "step": 300 + }, + { + "entropy": 0.6199766182899475, + "epoch": 0.8177570093457944, + "grad_norm": 0.38574621081352234, + "learning_rate": 0.00012590007874097354, + "loss": 0.5904315948486328, + "mean_token_accuracy": 0.8250177747011185, + "num_tokens": 1147400.0, + "step": 350 + }, + { + "entropy": 0.5950348073244095, + "epoch": 0.9345794392523364, + "grad_norm": 0.41016682982444763, + "learning_rate": 0.0001439373393055829, + "loss": 0.5652639389038085, + "mean_token_accuracy": 0.83076107442379, + "num_tokens": 1310490.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7618740917026222, + "eval_loss": 0.6691045761108398, + "eval_mean_token_accuracy": 0.8110752314716191, + "eval_num_tokens": 1399670.0, + "eval_runtime": 45.674, + "eval_samples_per_second": 26.952, + "eval_steps_per_second": 3.372, + "step": 428 + }, + { + "entropy": 0.5888169056177139, + "epoch": 1.0514018691588785, + "grad_norm": 0.339750200510025, + "learning_rate": 0.00015438762800645052, + "loss": 0.5587311935424805, + "mean_token_accuracy": 0.832102472782135, + "num_tokens": 1471726.0, + "step": 450 + }, + { + "entropy": 0.5644756412506103, + "epoch": 1.1682242990654206, + "grad_norm": 0.32897934317588806, + "learning_rate": 0.00015426955859477223, + "loss": 0.5355723571777343, + "mean_token_accuracy": 0.8378035259246827, + "num_tokens": 1638060.0, + "step": 500 + }, + { + "entropy": 0.5603723293542862, + "epoch": 1.2850467289719627, + "grad_norm": 0.3474414646625519, + "learning_rate": 0.00015402334678635756, + "loss": 0.5328482437133789, + "mean_token_accuracy": 0.8384119784832, + "num_tokens": 1802005.0, + "step": 550 + }, + { + "entropy": 0.551156692802906, + "epoch": 1.4018691588785046, + "grad_norm": 0.21533823013305664, + "learning_rate": 0.00015364940195109232, + "loss": 0.5259516525268555, + "mean_token_accuracy": 0.8401002615690232, + "num_tokens": 1969267.0, + "step": 600 + }, + { + "entropy": 0.5456148293614388, + "epoch": 1.5186915887850467, + "grad_norm": 0.3005809485912323, + "learning_rate": 0.0001531483458372002, + "loss": 0.5163665008544922, + "mean_token_accuracy": 0.8423654067516327, + "num_tokens": 2134793.0, + "step": 650 + }, + { + "entropy": 0.540266016125679, + "epoch": 1.6355140186915889, + "grad_norm": 0.2895597815513611, + "learning_rate": 0.00015252101153747827, + "loss": 0.5159417343139648, + "mean_token_accuracy": 0.8422912752628327, + "num_tokens": 2300672.0, + "step": 700 + }, + { + "entropy": 0.5450576081871986, + "epoch": 1.7523364485981308, + "grad_norm": 0.26751118898391724, + "learning_rate": 0.00015176844210413562, + "loss": 0.5150370788574219, + "mean_token_accuracy": 0.8433581227064133, + "num_tokens": 2459315.0, + "step": 750 + }, + { + "entropy": 0.540363709628582, + "epoch": 1.8691588785046729, + "grad_norm": 0.25945547223091125, + "learning_rate": 0.0001508918888145379, + "loss": 0.5133833312988281, + "mean_token_accuracy": 0.8442667639255523, + "num_tokens": 2616878.0, + "step": 800 + }, + { + "entropy": 0.5406239181756973, + "epoch": 1.985981308411215, + "grad_norm": 0.2768362760543823, + "learning_rate": 0.0001498928090907414, + "loss": 0.5116788864135742, + "mean_token_accuracy": 0.8440770989656449, + "num_tokens": 2778909.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6602114913138476, + "eval_loss": 0.6344706416130066, + "eval_mean_token_accuracy": 0.8167721358212557, + "eval_num_tokens": 2799340.0, + "eval_runtime": 45.6608, + "eval_samples_per_second": 26.96, + "eval_steps_per_second": 3.373, + "step": 856 + }, + { + "entropy": 0.47262389928102494, + "epoch": 2.102803738317757, + "grad_norm": 0.2535194158554077, + "learning_rate": 0.00014877286407627584, + "loss": 0.4428855514526367, + "mean_token_accuracy": 0.8600562286376953, + "num_tokens": 2951115.0, + "step": 900 + }, + { + "entropy": 0.47976606965065005, + "epoch": 2.2196261682242993, + "grad_norm": 0.30369797348976135, + "learning_rate": 0.00014753391587420475, + "loss": 0.44907299041748044, + "mean_token_accuracy": 0.8582676541805268, + "num_tokens": 3115016.0, + "step": 950 + }, + { + "entropy": 0.47275764375925067, + "epoch": 2.336448598130841, + "grad_norm": 0.3171652853488922, + "learning_rate": 0.0001461780244510558, + "loss": 0.44435993194580076, + "mean_token_accuracy": 0.8601345640420913, + "num_tokens": 3274313.0, + "step": 1000 + }, + { + "entropy": 0.4735608258843422, + "epoch": 2.453271028037383, + "grad_norm": 0.29764947295188904, + "learning_rate": 0.00014470744421176885, + "loss": 0.44539962768554686, + "mean_token_accuracy": 0.8607082539796829, + "num_tokens": 3440379.0, + "step": 1050 + }, + { + "entropy": 0.4744651636481285, + "epoch": 2.5700934579439254, + "grad_norm": 0.23777751624584198, + "learning_rate": 0.00014312462025135595, + "loss": 0.44422054290771484, + "mean_token_accuracy": 0.859566843509674, + "num_tokens": 3606755.0, + "step": 1100 + }, + { + "entropy": 0.47177853137254716, + "epoch": 2.6869158878504673, + "grad_norm": 0.28442806005477905, + "learning_rate": 0.00014143218428950668, + "loss": 0.4431512069702148, + "mean_token_accuracy": 0.8603039968013764, + "num_tokens": 3766555.0, + "step": 1150 + }, + { + "entropy": 0.4803256571292877, + "epoch": 2.803738317757009, + "grad_norm": 0.2728305757045746, + "learning_rate": 0.00013963295029489697, + "loss": 0.44966583251953124, + "mean_token_accuracy": 0.8587929052114487, + "num_tokens": 3927164.0, + "step": 1200 + }, + { + "entropy": 0.4743427449464798, + "epoch": 2.9205607476635516, + "grad_norm": 0.29242947697639465, + "learning_rate": 0.0001377299098064774, + "loss": 0.4463462829589844, + "mean_token_accuracy": 0.8596956437826156, + "num_tokens": 4090727.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.611924365356371, + "eval_loss": 0.6364901661872864, + "eval_mean_token_accuracy": 0.8110434765165503, + "eval_num_tokens": 4199010.0, + "eval_runtime": 45.6331, + "eval_samples_per_second": 26.976, + "eval_steps_per_second": 3.375, + "step": 1284 + }, + { + "entropy": 0.4497017902135849, + "epoch": 3.0373831775700935, + "grad_norm": 0.36632540822029114, + "learning_rate": 0.0001357262269595206, + "loss": 0.41871109008789065, + "mean_token_accuracy": 0.8677674496173858, + "num_tokens": 4252535.0, + "step": 1300 + }, + { + "entropy": 0.39628969103097916, + "epoch": 3.1542056074766354, + "grad_norm": 0.3463972210884094, + "learning_rate": 0.00013362523322469648, + "loss": 0.3633641815185547, + "mean_token_accuracy": 0.8810112345218658, + "num_tokens": 4414122.0, + "step": 1350 + }, + { + "entropy": 0.405142662525177, + "epoch": 3.2710280373831777, + "grad_norm": 0.3261754512786865, + "learning_rate": 0.00013143042186892402, + "loss": 0.37408100128173827, + "mean_token_accuracy": 0.8773768872022629, + "num_tokens": 4571859.0, + "step": 1400 + }, + { + "entropy": 0.3960475382208824, + "epoch": 3.3878504672897196, + "grad_norm": 0.32290324568748474, + "learning_rate": 0.00012914544214720838, + "loss": 0.3667728424072266, + "mean_token_accuracy": 0.8809277415275574, + "num_tokens": 4735006.0, + "step": 1450 + }, + { + "entropy": 0.39949932754039763, + "epoch": 3.5046728971962615, + "grad_norm": 0.3406212627887726, + "learning_rate": 0.0001267740932351207, + "loss": 0.37073043823242186, + "mean_token_accuracy": 0.879340250492096, + "num_tokens": 4898228.0, + "step": 1500 + }, + { + "entropy": 0.4047835928201675, + "epoch": 3.621495327102804, + "grad_norm": 0.30698826909065247, + "learning_rate": 0.00012432031791200892, + "loss": 0.3761457824707031, + "mean_token_accuracy": 0.8780743163824082, + "num_tokens": 5063413.0, + "step": 1550 + }, + { + "entropy": 0.4085030257701874, + "epoch": 3.7383177570093458, + "grad_norm": 0.30055665969848633, + "learning_rate": 0.00012178819600544269, + "loss": 0.379119873046875, + "mean_token_accuracy": 0.8756581097841263, + "num_tokens": 5227672.0, + "step": 1600 + }, + { + "entropy": 0.4101005423069, + "epoch": 3.8551401869158877, + "grad_norm": 0.32962530851364136, + "learning_rate": 0.00011918193760779132, + "loss": 0.3822117614746094, + "mean_token_accuracy": 0.8762752747535706, + "num_tokens": 5389834.0, + "step": 1650 + }, + { + "entropy": 0.40136953949928283, + "epoch": 3.97196261682243, + "grad_norm": 0.2512304186820984, + "learning_rate": 0.00011650587607621428, + "loss": 0.3733771514892578, + "mean_token_accuracy": 0.8783003056049347, + "num_tokens": 5559167.0, + "step": 1700 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5574058420085287, + "eval_loss": 0.6562352180480957, + "eval_mean_token_accuracy": 0.8172229067845778, + "eval_num_tokens": 5598680.0, + "eval_runtime": 45.6372, + "eval_samples_per_second": 26.974, + "eval_steps_per_second": 3.374, + "step": 1712 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.036135513426944e+18, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/README.md b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/adapter_config.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..368b2eaa099a70c44e0aa69e3cf7163ac54c47fa --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.030195310029781444, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "v_proj", + "down_proj", + "up_proj", + "k_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/chat_template.jinja b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/tokenizer_config.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/trainer_state.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3ada94bd7d0597db85504dc74725cfa483c64440 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/trainer_state.json @@ -0,0 +1,509 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2140, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6439054441452026, + "epoch": 0.11682242990654206, + "grad_norm": 0.49546942114830017, + "learning_rate": 1.7676515353317202e-05, + "loss": 1.6080712890625, + "mean_token_accuracy": 0.6552277088165284, + "num_tokens": 162248.0, + "step": 50 + }, + { + "entropy": 0.8895833045244217, + "epoch": 0.2336448598130841, + "grad_norm": 0.6087712645530701, + "learning_rate": 3.571377591792659e-05, + "loss": 0.8471942138671875, + "mean_token_accuracy": 0.7710993754863739, + "num_tokens": 327614.0, + "step": 100 + }, + { + "entropy": 0.7270394039154052, + "epoch": 0.35046728971962615, + "grad_norm": 0.4936729371547699, + "learning_rate": 5.375103648253598e-05, + "loss": 0.6940426635742187, + "mean_token_accuracy": 0.7985459506511688, + "num_tokens": 494582.0, + "step": 150 + }, + { + "entropy": 0.6873268783092499, + "epoch": 0.4672897196261682, + "grad_norm": 0.5290274620056152, + "learning_rate": 7.178829704714537e-05, + "loss": 0.6487248229980469, + "mean_token_accuracy": 0.8118354797363281, + "num_tokens": 653981.0, + "step": 200 + }, + { + "entropy": 0.6600921255350113, + "epoch": 0.5841121495327103, + "grad_norm": 0.3852632939815521, + "learning_rate": 8.982555761175476e-05, + "loss": 0.6241029357910156, + "mean_token_accuracy": 0.8171858608722686, + "num_tokens": 814840.0, + "step": 250 + }, + { + "entropy": 0.6286565268039703, + "epoch": 0.7009345794392523, + "grad_norm": 0.44116389751434326, + "learning_rate": 0.00010786281817636416, + "loss": 0.5951404190063476, + "mean_token_accuracy": 0.8241004174947739, + "num_tokens": 979542.0, + "step": 300 + }, + { + "entropy": 0.6199766182899475, + "epoch": 0.8177570093457944, + "grad_norm": 0.38574621081352234, + "learning_rate": 0.00012590007874097354, + "loss": 0.5904315948486328, + "mean_token_accuracy": 0.8250177747011185, + "num_tokens": 1147400.0, + "step": 350 + }, + { + "entropy": 0.5950348073244095, + "epoch": 0.9345794392523364, + "grad_norm": 0.41016682982444763, + "learning_rate": 0.0001439373393055829, + "loss": 0.5652639389038085, + "mean_token_accuracy": 0.83076107442379, + "num_tokens": 1310490.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7618740917026222, + "eval_loss": 0.6691045761108398, + "eval_mean_token_accuracy": 0.8110752314716191, + "eval_num_tokens": 1399670.0, + "eval_runtime": 45.674, + "eval_samples_per_second": 26.952, + "eval_steps_per_second": 3.372, + "step": 428 + }, + { + "entropy": 0.5888169056177139, + "epoch": 1.0514018691588785, + "grad_norm": 0.339750200510025, + "learning_rate": 0.00015438762800645052, + "loss": 0.5587311935424805, + "mean_token_accuracy": 0.832102472782135, + "num_tokens": 1471726.0, + "step": 450 + }, + { + "entropy": 0.5644756412506103, + "epoch": 1.1682242990654206, + "grad_norm": 0.32897934317588806, + "learning_rate": 0.00015426955859477223, + "loss": 0.5355723571777343, + "mean_token_accuracy": 0.8378035259246827, + "num_tokens": 1638060.0, + "step": 500 + }, + { + "entropy": 0.5603723293542862, + "epoch": 1.2850467289719627, + "grad_norm": 0.3474414646625519, + "learning_rate": 0.00015402334678635756, + "loss": 0.5328482437133789, + "mean_token_accuracy": 0.8384119784832, + "num_tokens": 1802005.0, + "step": 550 + }, + { + "entropy": 0.551156692802906, + "epoch": 1.4018691588785046, + "grad_norm": 0.21533823013305664, + "learning_rate": 0.00015364940195109232, + "loss": 0.5259516525268555, + "mean_token_accuracy": 0.8401002615690232, + "num_tokens": 1969267.0, + "step": 600 + }, + { + "entropy": 0.5456148293614388, + "epoch": 1.5186915887850467, + "grad_norm": 0.3005809485912323, + "learning_rate": 0.0001531483458372002, + "loss": 0.5163665008544922, + "mean_token_accuracy": 0.8423654067516327, + "num_tokens": 2134793.0, + "step": 650 + }, + { + "entropy": 0.540266016125679, + "epoch": 1.6355140186915889, + "grad_norm": 0.2895597815513611, + "learning_rate": 0.00015252101153747827, + "loss": 0.5159417343139648, + "mean_token_accuracy": 0.8422912752628327, + "num_tokens": 2300672.0, + "step": 700 + }, + { + "entropy": 0.5450576081871986, + "epoch": 1.7523364485981308, + "grad_norm": 0.26751118898391724, + "learning_rate": 0.00015176844210413562, + "loss": 0.5150370788574219, + "mean_token_accuracy": 0.8433581227064133, + "num_tokens": 2459315.0, + "step": 750 + }, + { + "entropy": 0.540363709628582, + "epoch": 1.8691588785046729, + "grad_norm": 0.25945547223091125, + "learning_rate": 0.0001508918888145379, + "loss": 0.5133833312988281, + "mean_token_accuracy": 0.8442667639255523, + "num_tokens": 2616878.0, + "step": 800 + }, + { + "entropy": 0.5406239181756973, + "epoch": 1.985981308411215, + "grad_norm": 0.2768362760543823, + "learning_rate": 0.0001498928090907414, + "loss": 0.5116788864135742, + "mean_token_accuracy": 0.8440770989656449, + "num_tokens": 2778909.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6602114913138476, + "eval_loss": 0.6344706416130066, + "eval_mean_token_accuracy": 0.8167721358212557, + "eval_num_tokens": 2799340.0, + "eval_runtime": 45.6608, + "eval_samples_per_second": 26.96, + "eval_steps_per_second": 3.373, + "step": 856 + }, + { + "entropy": 0.47262389928102494, + "epoch": 2.102803738317757, + "grad_norm": 0.2535194158554077, + "learning_rate": 0.00014877286407627584, + "loss": 0.4428855514526367, + "mean_token_accuracy": 0.8600562286376953, + "num_tokens": 2951115.0, + "step": 900 + }, + { + "entropy": 0.47976606965065005, + "epoch": 2.2196261682242993, + "grad_norm": 0.30369797348976135, + "learning_rate": 0.00014753391587420475, + "loss": 0.44907299041748044, + "mean_token_accuracy": 0.8582676541805268, + "num_tokens": 3115016.0, + "step": 950 + }, + { + "entropy": 0.47275764375925067, + "epoch": 2.336448598130841, + "grad_norm": 0.3171652853488922, + "learning_rate": 0.0001461780244510558, + "loss": 0.44435993194580076, + "mean_token_accuracy": 0.8601345640420913, + "num_tokens": 3274313.0, + "step": 1000 + }, + { + "entropy": 0.4735608258843422, + "epoch": 2.453271028037383, + "grad_norm": 0.29764947295188904, + "learning_rate": 0.00014470744421176885, + "loss": 0.44539962768554686, + "mean_token_accuracy": 0.8607082539796829, + "num_tokens": 3440379.0, + "step": 1050 + }, + { + "entropy": 0.4744651636481285, + "epoch": 2.5700934579439254, + "grad_norm": 0.23777751624584198, + "learning_rate": 0.00014312462025135595, + "loss": 0.44422054290771484, + "mean_token_accuracy": 0.859566843509674, + "num_tokens": 3606755.0, + "step": 1100 + }, + { + "entropy": 0.47177853137254716, + "epoch": 2.6869158878504673, + "grad_norm": 0.28442806005477905, + "learning_rate": 0.00014143218428950668, + "loss": 0.4431512069702148, + "mean_token_accuracy": 0.8603039968013764, + "num_tokens": 3766555.0, + "step": 1150 + }, + { + "entropy": 0.4803256571292877, + "epoch": 2.803738317757009, + "grad_norm": 0.2728305757045746, + "learning_rate": 0.00013963295029489697, + "loss": 0.44966583251953124, + "mean_token_accuracy": 0.8587929052114487, + "num_tokens": 3927164.0, + "step": 1200 + }, + { + "entropy": 0.4743427449464798, + "epoch": 2.9205607476635516, + "grad_norm": 0.29242947697639465, + "learning_rate": 0.0001377299098064774, + "loss": 0.4463462829589844, + "mean_token_accuracy": 0.8596956437826156, + "num_tokens": 4090727.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.611924365356371, + "eval_loss": 0.6364901661872864, + "eval_mean_token_accuracy": 0.8110434765165503, + "eval_num_tokens": 4199010.0, + "eval_runtime": 45.6331, + "eval_samples_per_second": 26.976, + "eval_steps_per_second": 3.375, + "step": 1284 + }, + { + "entropy": 0.4497017902135849, + "epoch": 3.0373831775700935, + "grad_norm": 0.36632540822029114, + "learning_rate": 0.0001357262269595206, + "loss": 0.41871109008789065, + "mean_token_accuracy": 0.8677674496173858, + "num_tokens": 4252535.0, + "step": 1300 + }, + { + "entropy": 0.39628969103097916, + "epoch": 3.1542056074766354, + "grad_norm": 0.3463972210884094, + "learning_rate": 0.00013362523322469648, + "loss": 0.3633641815185547, + "mean_token_accuracy": 0.8810112345218658, + "num_tokens": 4414122.0, + "step": 1350 + }, + { + "entropy": 0.405142662525177, + "epoch": 3.2710280373831777, + "grad_norm": 0.3261754512786865, + "learning_rate": 0.00013143042186892402, + "loss": 0.37408100128173827, + "mean_token_accuracy": 0.8773768872022629, + "num_tokens": 4571859.0, + "step": 1400 + }, + { + "entropy": 0.3960475382208824, + "epoch": 3.3878504672897196, + "grad_norm": 0.32290324568748474, + "learning_rate": 0.00012914544214720838, + "loss": 0.3667728424072266, + "mean_token_accuracy": 0.8809277415275574, + "num_tokens": 4735006.0, + "step": 1450 + }, + { + "entropy": 0.39949932754039763, + "epoch": 3.5046728971962615, + "grad_norm": 0.3406212627887726, + "learning_rate": 0.0001267740932351207, + "loss": 0.37073043823242186, + "mean_token_accuracy": 0.879340250492096, + "num_tokens": 4898228.0, + "step": 1500 + }, + { + "entropy": 0.4047835928201675, + "epoch": 3.621495327102804, + "grad_norm": 0.30698826909065247, + "learning_rate": 0.00012432031791200892, + "loss": 0.3761457824707031, + "mean_token_accuracy": 0.8780743163824082, + "num_tokens": 5063413.0, + "step": 1550 + }, + { + "entropy": 0.4085030257701874, + "epoch": 3.7383177570093458, + "grad_norm": 0.30055665969848633, + "learning_rate": 0.00012178819600544269, + "loss": 0.379119873046875, + "mean_token_accuracy": 0.8756581097841263, + "num_tokens": 5227672.0, + "step": 1600 + }, + { + "entropy": 0.4101005423069, + "epoch": 3.8551401869158877, + "grad_norm": 0.32962530851364136, + "learning_rate": 0.00011918193760779132, + "loss": 0.3822117614746094, + "mean_token_accuracy": 0.8762752747535706, + "num_tokens": 5389834.0, + "step": 1650 + }, + { + "entropy": 0.40136953949928283, + "epoch": 3.97196261682243, + "grad_norm": 0.2512304186820984, + "learning_rate": 0.00011650587607621428, + "loss": 0.3733771514892578, + "mean_token_accuracy": 0.8783003056049347, + "num_tokens": 5559167.0, + "step": 1700 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5574058420085287, + "eval_loss": 0.6562352180480957, + "eval_mean_token_accuracy": 0.8172229067845778, + "eval_num_tokens": 5598680.0, + "eval_runtime": 45.6372, + "eval_samples_per_second": 26.974, + "eval_steps_per_second": 3.374, + "step": 1712 + }, + { + "entropy": 0.33769124254584315, + "epoch": 4.088785046728972, + "grad_norm": 0.35916072130203247, + "learning_rate": 0.00011376446082770247, + "loss": 0.3022454833984375, + "mean_token_accuracy": 0.898470618724823, + "num_tokens": 5721701.0, + "step": 1750 + }, + { + "entropy": 0.3153342293202877, + "epoch": 4.205607476635514, + "grad_norm": 0.4674874544143677, + "learning_rate": 0.00011096224994114971, + "loss": 0.28416419982910157, + "mean_token_accuracy": 0.9044169551134109, + "num_tokens": 5884163.0, + "step": 1800 + }, + { + "entropy": 0.32269440904259683, + "epoch": 4.322429906542056, + "grad_norm": 0.4200516641139984, + "learning_rate": 0.00010810390257875522, + "loss": 0.29085651397705076, + "mean_token_accuracy": 0.9013593572378159, + "num_tokens": 6053869.0, + "step": 1850 + }, + { + "entropy": 0.324297761619091, + "epoch": 4.4392523364485985, + "grad_norm": 0.35716596245765686, + "learning_rate": 0.0001051941712393573, + "loss": 0.2931093406677246, + "mean_token_accuracy": 0.9010413753986358, + "num_tokens": 6217351.0, + "step": 1900 + }, + { + "entropy": 0.3256940242648125, + "epoch": 4.55607476635514, + "grad_norm": 0.35059621930122375, + "learning_rate": 0.00010223789385657872, + "loss": 0.29291555404663083, + "mean_token_accuracy": 0.9013585340976715, + "num_tokens": 6379796.0, + "step": 1950 + }, + { + "entropy": 0.32556063055992124, + "epoch": 4.672897196261682, + "grad_norm": 0.31053468585014343, + "learning_rate": 9.923998575492214e-05, + "loss": 0.29418453216552737, + "mean_token_accuracy": 0.9006082111597061, + "num_tokens": 6550276.0, + "step": 2000 + }, + { + "entropy": 0.3271982085704803, + "epoch": 4.789719626168225, + "grad_norm": 0.3549234867095947, + "learning_rate": 9.62054314771894e-05, + "loss": 0.293514575958252, + "mean_token_accuracy": 0.9009167104959488, + "num_tokens": 6709429.0, + "step": 2050 + }, + { + "entropy": 0.32230626583099364, + "epoch": 4.906542056074766, + "grad_norm": 0.36930036544799805, + "learning_rate": 9.313927649681362e-05, + "loss": 0.29048364639282226, + "mean_token_accuracy": 0.9017211717367172, + "num_tokens": 6870956.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5097629334051887, + "eval_loss": 0.7178825736045837, + "eval_mean_token_accuracy": 0.8133898025209253, + "eval_num_tokens": 6998350.0, + "eval_runtime": 45.6467, + "eval_samples_per_second": 26.968, + "eval_steps_per_second": 3.374, + "step": 2140 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.296106037517312e+18, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/README.md b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/adapter_config.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..368b2eaa099a70c44e0aa69e3cf7163ac54c47fa --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.030195310029781444, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "v_proj", + "down_proj", + "up_proj", + "k_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/chat_template.jinja b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/tokenizer_config.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/trainer_state.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3ecf90ed1012b7b7b86e99a822e0c0fd5698f2ad --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/trainer_state.json @@ -0,0 +1,610 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2568, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6439054441452026, + "epoch": 0.11682242990654206, + "grad_norm": 0.49546942114830017, + "learning_rate": 1.7676515353317202e-05, + "loss": 1.6080712890625, + "mean_token_accuracy": 0.6552277088165284, + "num_tokens": 162248.0, + "step": 50 + }, + { + "entropy": 0.8895833045244217, + "epoch": 0.2336448598130841, + "grad_norm": 0.6087712645530701, + "learning_rate": 3.571377591792659e-05, + "loss": 0.8471942138671875, + "mean_token_accuracy": 0.7710993754863739, + "num_tokens": 327614.0, + "step": 100 + }, + { + "entropy": 0.7270394039154052, + "epoch": 0.35046728971962615, + "grad_norm": 0.4936729371547699, + "learning_rate": 5.375103648253598e-05, + "loss": 0.6940426635742187, + "mean_token_accuracy": 0.7985459506511688, + "num_tokens": 494582.0, + "step": 150 + }, + { + "entropy": 0.6873268783092499, + "epoch": 0.4672897196261682, + "grad_norm": 0.5290274620056152, + "learning_rate": 7.178829704714537e-05, + "loss": 0.6487248229980469, + "mean_token_accuracy": 0.8118354797363281, + "num_tokens": 653981.0, + "step": 200 + }, + { + "entropy": 0.6600921255350113, + "epoch": 0.5841121495327103, + "grad_norm": 0.3852632939815521, + "learning_rate": 8.982555761175476e-05, + "loss": 0.6241029357910156, + "mean_token_accuracy": 0.8171858608722686, + "num_tokens": 814840.0, + "step": 250 + }, + { + "entropy": 0.6286565268039703, + "epoch": 0.7009345794392523, + "grad_norm": 0.44116389751434326, + "learning_rate": 0.00010786281817636416, + "loss": 0.5951404190063476, + "mean_token_accuracy": 0.8241004174947739, + "num_tokens": 979542.0, + "step": 300 + }, + { + "entropy": 0.6199766182899475, + "epoch": 0.8177570093457944, + "grad_norm": 0.38574621081352234, + "learning_rate": 0.00012590007874097354, + "loss": 0.5904315948486328, + "mean_token_accuracy": 0.8250177747011185, + "num_tokens": 1147400.0, + "step": 350 + }, + { + "entropy": 0.5950348073244095, + "epoch": 0.9345794392523364, + "grad_norm": 0.41016682982444763, + "learning_rate": 0.0001439373393055829, + "loss": 0.5652639389038085, + "mean_token_accuracy": 0.83076107442379, + "num_tokens": 1310490.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7618740917026222, + "eval_loss": 0.6691045761108398, + "eval_mean_token_accuracy": 0.8110752314716191, + "eval_num_tokens": 1399670.0, + "eval_runtime": 45.674, + "eval_samples_per_second": 26.952, + "eval_steps_per_second": 3.372, + "step": 428 + }, + { + "entropy": 0.5888169056177139, + "epoch": 1.0514018691588785, + "grad_norm": 0.339750200510025, + "learning_rate": 0.00015438762800645052, + "loss": 0.5587311935424805, + "mean_token_accuracy": 0.832102472782135, + "num_tokens": 1471726.0, + "step": 450 + }, + { + "entropy": 0.5644756412506103, + "epoch": 1.1682242990654206, + "grad_norm": 0.32897934317588806, + "learning_rate": 0.00015426955859477223, + "loss": 0.5355723571777343, + "mean_token_accuracy": 0.8378035259246827, + "num_tokens": 1638060.0, + "step": 500 + }, + { + "entropy": 0.5603723293542862, + "epoch": 1.2850467289719627, + "grad_norm": 0.3474414646625519, + "learning_rate": 0.00015402334678635756, + "loss": 0.5328482437133789, + "mean_token_accuracy": 0.8384119784832, + "num_tokens": 1802005.0, + "step": 550 + }, + { + "entropy": 0.551156692802906, + "epoch": 1.4018691588785046, + "grad_norm": 0.21533823013305664, + "learning_rate": 0.00015364940195109232, + "loss": 0.5259516525268555, + "mean_token_accuracy": 0.8401002615690232, + "num_tokens": 1969267.0, + "step": 600 + }, + { + "entropy": 0.5456148293614388, + "epoch": 1.5186915887850467, + "grad_norm": 0.3005809485912323, + "learning_rate": 0.0001531483458372002, + "loss": 0.5163665008544922, + "mean_token_accuracy": 0.8423654067516327, + "num_tokens": 2134793.0, + "step": 650 + }, + { + "entropy": 0.540266016125679, + "epoch": 1.6355140186915889, + "grad_norm": 0.2895597815513611, + "learning_rate": 0.00015252101153747827, + "loss": 0.5159417343139648, + "mean_token_accuracy": 0.8422912752628327, + "num_tokens": 2300672.0, + "step": 700 + }, + { + "entropy": 0.5450576081871986, + "epoch": 1.7523364485981308, + "grad_norm": 0.26751118898391724, + "learning_rate": 0.00015176844210413562, + "loss": 0.5150370788574219, + "mean_token_accuracy": 0.8433581227064133, + "num_tokens": 2459315.0, + "step": 750 + }, + { + "entropy": 0.540363709628582, + "epoch": 1.8691588785046729, + "grad_norm": 0.25945547223091125, + "learning_rate": 0.0001508918888145379, + "loss": 0.5133833312988281, + "mean_token_accuracy": 0.8442667639255523, + "num_tokens": 2616878.0, + "step": 800 + }, + { + "entropy": 0.5406239181756973, + "epoch": 1.985981308411215, + "grad_norm": 0.2768362760543823, + "learning_rate": 0.0001498928090907414, + "loss": 0.5116788864135742, + "mean_token_accuracy": 0.8440770989656449, + "num_tokens": 2778909.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6602114913138476, + "eval_loss": 0.6344706416130066, + "eval_mean_token_accuracy": 0.8167721358212557, + "eval_num_tokens": 2799340.0, + "eval_runtime": 45.6608, + "eval_samples_per_second": 26.96, + "eval_steps_per_second": 3.373, + "step": 856 + }, + { + "entropy": 0.47262389928102494, + "epoch": 2.102803738317757, + "grad_norm": 0.2535194158554077, + "learning_rate": 0.00014877286407627584, + "loss": 0.4428855514526367, + "mean_token_accuracy": 0.8600562286376953, + "num_tokens": 2951115.0, + "step": 900 + }, + { + "entropy": 0.47976606965065005, + "epoch": 2.2196261682242993, + "grad_norm": 0.30369797348976135, + "learning_rate": 0.00014753391587420475, + "loss": 0.44907299041748044, + "mean_token_accuracy": 0.8582676541805268, + "num_tokens": 3115016.0, + "step": 950 + }, + { + "entropy": 0.47275764375925067, + "epoch": 2.336448598130841, + "grad_norm": 0.3171652853488922, + "learning_rate": 0.0001461780244510558, + "loss": 0.44435993194580076, + "mean_token_accuracy": 0.8601345640420913, + "num_tokens": 3274313.0, + "step": 1000 + }, + { + "entropy": 0.4735608258843422, + "epoch": 2.453271028037383, + "grad_norm": 0.29764947295188904, + "learning_rate": 0.00014470744421176885, + "loss": 0.44539962768554686, + "mean_token_accuracy": 0.8607082539796829, + "num_tokens": 3440379.0, + "step": 1050 + }, + { + "entropy": 0.4744651636481285, + "epoch": 2.5700934579439254, + "grad_norm": 0.23777751624584198, + "learning_rate": 0.00014312462025135595, + "loss": 0.44422054290771484, + "mean_token_accuracy": 0.859566843509674, + "num_tokens": 3606755.0, + "step": 1100 + }, + { + "entropy": 0.47177853137254716, + "epoch": 2.6869158878504673, + "grad_norm": 0.28442806005477905, + "learning_rate": 0.00014143218428950668, + "loss": 0.4431512069702148, + "mean_token_accuracy": 0.8603039968013764, + "num_tokens": 3766555.0, + "step": 1150 + }, + { + "entropy": 0.4803256571292877, + "epoch": 2.803738317757009, + "grad_norm": 0.2728305757045746, + "learning_rate": 0.00013963295029489697, + "loss": 0.44966583251953124, + "mean_token_accuracy": 0.8587929052114487, + "num_tokens": 3927164.0, + "step": 1200 + }, + { + "entropy": 0.4743427449464798, + "epoch": 2.9205607476635516, + "grad_norm": 0.29242947697639465, + "learning_rate": 0.0001377299098064774, + "loss": 0.4463462829589844, + "mean_token_accuracy": 0.8596956437826156, + "num_tokens": 4090727.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.611924365356371, + "eval_loss": 0.6364901661872864, + "eval_mean_token_accuracy": 0.8110434765165503, + "eval_num_tokens": 4199010.0, + "eval_runtime": 45.6331, + "eval_samples_per_second": 26.976, + "eval_steps_per_second": 3.375, + "step": 1284 + }, + { + "entropy": 0.4497017902135849, + "epoch": 3.0373831775700935, + "grad_norm": 0.36632540822029114, + "learning_rate": 0.0001357262269595206, + "loss": 0.41871109008789065, + "mean_token_accuracy": 0.8677674496173858, + "num_tokens": 4252535.0, + "step": 1300 + }, + { + "entropy": 0.39628969103097916, + "epoch": 3.1542056074766354, + "grad_norm": 0.3463972210884094, + "learning_rate": 0.00013362523322469648, + "loss": 0.3633641815185547, + "mean_token_accuracy": 0.8810112345218658, + "num_tokens": 4414122.0, + "step": 1350 + }, + { + "entropy": 0.405142662525177, + "epoch": 3.2710280373831777, + "grad_norm": 0.3261754512786865, + "learning_rate": 0.00013143042186892402, + "loss": 0.37408100128173827, + "mean_token_accuracy": 0.8773768872022629, + "num_tokens": 4571859.0, + "step": 1400 + }, + { + "entropy": 0.3960475382208824, + "epoch": 3.3878504672897196, + "grad_norm": 0.32290324568748474, + "learning_rate": 0.00012914544214720838, + "loss": 0.3667728424072266, + "mean_token_accuracy": 0.8809277415275574, + "num_tokens": 4735006.0, + "step": 1450 + }, + { + "entropy": 0.39949932754039763, + "epoch": 3.5046728971962615, + "grad_norm": 0.3406212627887726, + "learning_rate": 0.0001267740932351207, + "loss": 0.37073043823242186, + "mean_token_accuracy": 0.879340250492096, + "num_tokens": 4898228.0, + "step": 1500 + }, + { + "entropy": 0.4047835928201675, + "epoch": 3.621495327102804, + "grad_norm": 0.30698826909065247, + "learning_rate": 0.00012432031791200892, + "loss": 0.3761457824707031, + "mean_token_accuracy": 0.8780743163824082, + "num_tokens": 5063413.0, + "step": 1550 + }, + { + "entropy": 0.4085030257701874, + "epoch": 3.7383177570093458, + "grad_norm": 0.30055665969848633, + "learning_rate": 0.00012178819600544269, + "loss": 0.379119873046875, + "mean_token_accuracy": 0.8756581097841263, + "num_tokens": 5227672.0, + "step": 1600 + }, + { + "entropy": 0.4101005423069, + "epoch": 3.8551401869158877, + "grad_norm": 0.32962530851364136, + "learning_rate": 0.00011918193760779132, + "loss": 0.3822117614746094, + "mean_token_accuracy": 0.8762752747535706, + "num_tokens": 5389834.0, + "step": 1650 + }, + { + "entropy": 0.40136953949928283, + "epoch": 3.97196261682243, + "grad_norm": 0.2512304186820984, + "learning_rate": 0.00011650587607621428, + "loss": 0.3733771514892578, + "mean_token_accuracy": 0.8783003056049347, + "num_tokens": 5559167.0, + "step": 1700 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5574058420085287, + "eval_loss": 0.6562352180480957, + "eval_mean_token_accuracy": 0.8172229067845778, + "eval_num_tokens": 5598680.0, + "eval_runtime": 45.6372, + "eval_samples_per_second": 26.974, + "eval_steps_per_second": 3.374, + "step": 1712 + }, + { + "entropy": 0.33769124254584315, + "epoch": 4.088785046728972, + "grad_norm": 0.35916072130203247, + "learning_rate": 0.00011376446082770247, + "loss": 0.3022454833984375, + "mean_token_accuracy": 0.898470618724823, + "num_tokens": 5721701.0, + "step": 1750 + }, + { + "entropy": 0.3153342293202877, + "epoch": 4.205607476635514, + "grad_norm": 0.4674874544143677, + "learning_rate": 0.00011096224994114971, + "loss": 0.28416419982910157, + "mean_token_accuracy": 0.9044169551134109, + "num_tokens": 5884163.0, + "step": 1800 + }, + { + "entropy": 0.32269440904259683, + "epoch": 4.322429906542056, + "grad_norm": 0.4200516641139984, + "learning_rate": 0.00010810390257875522, + "loss": 0.29085651397705076, + "mean_token_accuracy": 0.9013593572378159, + "num_tokens": 6053869.0, + "step": 1850 + }, + { + "entropy": 0.324297761619091, + "epoch": 4.4392523364485985, + "grad_norm": 0.35716596245765686, + "learning_rate": 0.0001051941712393573, + "loss": 0.2931093406677246, + "mean_token_accuracy": 0.9010413753986358, + "num_tokens": 6217351.0, + "step": 1900 + }, + { + "entropy": 0.3256940242648125, + "epoch": 4.55607476635514, + "grad_norm": 0.35059621930122375, + "learning_rate": 0.00010223789385657872, + "loss": 0.29291555404663083, + "mean_token_accuracy": 0.9013585340976715, + "num_tokens": 6379796.0, + "step": 1950 + }, + { + "entropy": 0.32556063055992124, + "epoch": 4.672897196261682, + "grad_norm": 0.31053468585014343, + "learning_rate": 9.923998575492214e-05, + "loss": 0.29418453216552737, + "mean_token_accuracy": 0.9006082111597061, + "num_tokens": 6550276.0, + "step": 2000 + }, + { + "entropy": 0.3271982085704803, + "epoch": 4.789719626168225, + "grad_norm": 0.3549234867095947, + "learning_rate": 9.62054314771894e-05, + "loss": 0.293514575958252, + "mean_token_accuracy": 0.9009167104959488, + "num_tokens": 6709429.0, + "step": 2050 + }, + { + "entropy": 0.32230626583099364, + "epoch": 4.906542056074766, + "grad_norm": 0.36930036544799805, + "learning_rate": 9.313927649681362e-05, + "loss": 0.29048364639282226, + "mean_token_accuracy": 0.9017211717367172, + "num_tokens": 6870956.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5097629334051887, + "eval_loss": 0.7178825736045837, + "eval_mean_token_accuracy": 0.8133898025209253, + "eval_num_tokens": 6998350.0, + "eval_runtime": 45.6467, + "eval_samples_per_second": 26.968, + "eval_steps_per_second": 3.374, + "step": 2140 + }, + { + "entropy": 0.30689359933137894, + "epoch": 5.0233644859813085, + "grad_norm": 0.6219972372055054, + "learning_rate": 9.004661882888355e-05, + "loss": 0.27355791091918946, + "mean_token_accuracy": 0.9078934907913208, + "num_tokens": 7034085.0, + "step": 2150 + }, + { + "entropy": 0.23060906335711479, + "epoch": 5.140186915887851, + "grad_norm": 0.39721137285232544, + "learning_rate": 8.69326005538081e-05, + "loss": 0.19190786361694337, + "mean_token_accuracy": 0.9332777208089829, + "num_tokens": 7198745.0, + "step": 2200 + }, + { + "entropy": 0.23536642283201217, + "epoch": 5.257009345794392, + "grad_norm": 0.4206141233444214, + "learning_rate": 8.380239926771483e-05, + "loss": 0.19819820404052735, + "mean_token_accuracy": 0.9311726182699204, + "num_tokens": 7366588.0, + "step": 2250 + }, + { + "entropy": 0.2427162781357765, + "epoch": 5.373831775700935, + "grad_norm": 0.543883204460144, + "learning_rate": 8.066121947379727e-05, + "loss": 0.20555742263793944, + "mean_token_accuracy": 0.9283805179595948, + "num_tokens": 7531696.0, + "step": 2300 + }, + { + "entropy": 0.24081297397613524, + "epoch": 5.490654205607477, + "grad_norm": 0.4346058666706085, + "learning_rate": 7.751428392892456e-05, + "loss": 0.20357828140258788, + "mean_token_accuracy": 0.9291914713382721, + "num_tokens": 7688960.0, + "step": 2350 + }, + { + "entropy": 0.2414954523742199, + "epoch": 5.607476635514018, + "grad_norm": 0.48998570442199707, + "learning_rate": 7.43668249599014e-05, + "loss": 0.20533323287963867, + "mean_token_accuracy": 0.9290326738357544, + "num_tokens": 7851053.0, + "step": 2400 + }, + { + "entropy": 0.24239167034626008, + "epoch": 5.724299065420561, + "grad_norm": 0.5171953439712524, + "learning_rate": 7.122407576381592e-05, + "loss": 0.20603450775146484, + "mean_token_accuracy": 0.9289826327562332, + "num_tokens": 8001524.0, + "step": 2450 + }, + { + "entropy": 0.24068919852375983, + "epoch": 5.841121495327103, + "grad_norm": 0.5036585330963135, + "learning_rate": 6.809126170694096e-05, + "loss": 0.20551023483276368, + "mean_token_accuracy": 0.9283198803663254, + "num_tokens": 8166394.0, + "step": 2500 + }, + { + "entropy": 0.23394863292574883, + "epoch": 5.957943925233645, + "grad_norm": 0.4463285505771637, + "learning_rate": 6.497359163665513e-05, + "loss": 0.20060670852661133, + "mean_token_accuracy": 0.9296612071990967, + "num_tokens": 8339045.0, + "step": 2550 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4407621825283224, + "eval_loss": 0.7988200187683105, + "eval_mean_token_accuracy": 0.810667238839261, + "eval_num_tokens": 8398020.0, + "eval_runtime": 45.6554, + "eval_samples_per_second": 26.963, + "eval_steps_per_second": 3.373, + "step": 2568 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.55513965942272e+18, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/README.md b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/adapter_config.json b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..368b2eaa099a70c44e0aa69e3cf7163ac54c47fa --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.030195310029781444, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "v_proj", + "down_proj", + "up_proj", + "k_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/chat_template.jinja b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3-14B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c8f5c14ea435f3ac7dfbe30a0c5791ad094c54d4 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/trainer_state.json @@ -0,0 +1,139 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.24539877300613497, + "eval_steps": 20, + "global_step": 100, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.7432787808256e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..73c6d29bf12ce1525c866861d68e4beab2a05cb2 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/trainer_state.json @@ -0,0 +1,160 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.294478527607362, + "eval_steps": 20, + "global_step": 120, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.8615433449472e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b986f59a3f208c3c8dc649ecf4a7d7dbd70ee0e8 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/trainer_state.json @@ -0,0 +1,181 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.34355828220858897, + "eval_steps": 20, + "global_step": 140, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.9723537238016e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8d5fb0bdeec3ea219da76d86321b4cc4b1c04c41 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/trainer_state.json @@ -0,0 +1,202 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.39263803680981596, + "eval_steps": 20, + "global_step": 160, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.1601678880768e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a6d1cc59ee0e452df10131d3f741e8aeec0d8d5b --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/trainer_state.json @@ -0,0 +1,223 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.44171779141104295, + "eval_steps": 20, + "global_step": 180, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.01651151587328e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d5d327197989569158b44b651d1dbf115fc4dc32 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/trainer_state.json @@ -0,0 +1,55 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.049079754601226995, + "eval_steps": 20, + "global_step": 20, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.1376864780288e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..123cd2973ea752f944298b1867474d10a2289793 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/trainer_state.json @@ -0,0 +1,244 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.49079754601226994, + "eval_steps": 20, + "global_step": 200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.11317656977408e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8449683f0372ccda72684e5e6c088b04329b4edc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/trainer_state.json @@ -0,0 +1,265 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5398773006134969, + "eval_steps": 20, + "global_step": 220, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.22628870217728e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..47218a65fc89a844d0fd03a690078f171fd06735 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/trainer_state.json @@ -0,0 +1,286 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.588957055214724, + "eval_steps": 20, + "global_step": 240, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.3319398106112e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4ba7cc3560dabfebd3d50441d42f4cd76ca22750 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/trainer_state.json @@ -0,0 +1,307 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6380368098159509, + "eval_steps": 20, + "global_step": 260, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.45350457878528e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ccd5388e85129c884a21cbf6245a94231493c644 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/trainer_state.json @@ -0,0 +1,328 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6871165644171779, + "eval_steps": 20, + "global_step": 280, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + }, + { + "entropy": 0.6266187071800232, + "epoch": 0.6871165644171779, + "grad_norm": 0.5539442300796509, + "learning_rate": 5.805141830292018e-05, + "loss": 0.5868237495422364, + "mean_token_accuracy": 0.8236475840210915, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6527906462124415, + "eval_loss": 0.5967546105384827, + "eval_mean_token_accuracy": 0.8212518399102348, + "eval_num_tokens": 760708.0, + "eval_runtime": 110.5069, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 280 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.5618227810304e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..020d268776704cdbdad776b867b297a80cece120 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/trainer_state.json @@ -0,0 +1,349 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7361963190184049, + "eval_steps": 20, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + }, + { + "entropy": 0.6266187071800232, + "epoch": 0.6871165644171779, + "grad_norm": 0.5539442300796509, + "learning_rate": 5.805141830292018e-05, + "loss": 0.5868237495422364, + "mean_token_accuracy": 0.8236475840210915, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6527906462124415, + "eval_loss": 0.5967546105384827, + "eval_mean_token_accuracy": 0.8212518399102348, + "eval_num_tokens": 760708.0, + "eval_runtime": 110.5069, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 280 + }, + { + "entropy": 0.6342157602310181, + "epoch": 0.7361963190184049, + "grad_norm": 0.4901776611804962, + "learning_rate": 5.574059258291945e-05, + "loss": 0.5924944877624512, + "mean_token_accuracy": 0.8226904451847077, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6531158692496164, + "eval_loss": 0.589684784412384, + "eval_mean_token_accuracy": 0.8269473876271929, + "eval_num_tokens": 816117.0, + "eval_runtime": 110.5437, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 300 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.67553671921664e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1879acc643636102bcd1dc4a31afd667b165a1ac --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/trainer_state.json @@ -0,0 +1,370 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7852760736196319, + "eval_steps": 20, + "global_step": 320, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + }, + { + "entropy": 0.6266187071800232, + "epoch": 0.6871165644171779, + "grad_norm": 0.5539442300796509, + "learning_rate": 5.805141830292018e-05, + "loss": 0.5868237495422364, + "mean_token_accuracy": 0.8236475840210915, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6527906462124415, + "eval_loss": 0.5967546105384827, + "eval_mean_token_accuracy": 0.8212518399102348, + "eval_num_tokens": 760708.0, + "eval_runtime": 110.5069, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 280 + }, + { + "entropy": 0.6342157602310181, + "epoch": 0.7361963190184049, + "grad_norm": 0.4901776611804962, + "learning_rate": 5.574059258291945e-05, + "loss": 0.5924944877624512, + "mean_token_accuracy": 0.8226904451847077, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6531158692496164, + "eval_loss": 0.589684784412384, + "eval_mean_token_accuracy": 0.8269473876271929, + "eval_num_tokens": 816117.0, + "eval_runtime": 110.5437, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 300 + }, + { + "entropy": 0.6304035305976867, + "epoch": 0.7852760736196319, + "grad_norm": 0.5012097954750061, + "learning_rate": 5.327688135467081e-05, + "loss": 0.5837351322174072, + "mean_token_accuracy": 0.8231772750616073, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6399370544297355, + "eval_loss": 0.587123692035675, + "eval_mean_token_accuracy": 0.8258680891990662, + "eval_num_tokens": 869286.0, + "eval_runtime": 110.5025, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 320 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.78536627462144e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1cdf7cdb7824fc23fe7490519bf7dbcc68efe64b --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/trainer_state.json @@ -0,0 +1,391 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8343558282208589, + "eval_steps": 20, + "global_step": 340, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + }, + { + "entropy": 0.6266187071800232, + "epoch": 0.6871165644171779, + "grad_norm": 0.5539442300796509, + "learning_rate": 5.805141830292018e-05, + "loss": 0.5868237495422364, + "mean_token_accuracy": 0.8236475840210915, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6527906462124415, + "eval_loss": 0.5967546105384827, + "eval_mean_token_accuracy": 0.8212518399102348, + "eval_num_tokens": 760708.0, + "eval_runtime": 110.5069, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 280 + }, + { + "entropy": 0.6342157602310181, + "epoch": 0.7361963190184049, + "grad_norm": 0.4901776611804962, + "learning_rate": 5.574059258291945e-05, + "loss": 0.5924944877624512, + "mean_token_accuracy": 0.8226904451847077, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6531158692496164, + "eval_loss": 0.589684784412384, + "eval_mean_token_accuracy": 0.8269473876271929, + "eval_num_tokens": 816117.0, + "eval_runtime": 110.5437, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 300 + }, + { + "entropy": 0.6304035305976867, + "epoch": 0.7852760736196319, + "grad_norm": 0.5012097954750061, + "learning_rate": 5.327688135467081e-05, + "loss": 0.5837351322174072, + "mean_token_accuracy": 0.8231772750616073, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6399370544297355, + "eval_loss": 0.587123692035675, + "eval_mean_token_accuracy": 0.8258680891990662, + "eval_num_tokens": 869286.0, + "eval_runtime": 110.5025, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 320 + }, + { + "entropy": 0.6135720565915108, + "epoch": 0.8343558282208589, + "grad_norm": 0.4748188555240631, + "learning_rate": 5.0678326930861606e-05, + "loss": 0.5775577068328858, + "mean_token_accuracy": 0.8291384443640709, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6397319528034755, + "eval_loss": 0.5828214287757874, + "eval_mean_token_accuracy": 0.8270208791324071, + "eval_num_tokens": 922579.0, + "eval_runtime": 110.5417, + "eval_samples_per_second": 12.629, + "eval_steps_per_second": 1.583, + "step": 340 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.90158317776896e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9706553bf1c16131dea3c6eed6e2be37c00c2bb4 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/trainer_state.json @@ -0,0 +1,412 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8834355828220859, + "eval_steps": 20, + "global_step": 360, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + }, + { + "entropy": 0.6266187071800232, + "epoch": 0.6871165644171779, + "grad_norm": 0.5539442300796509, + "learning_rate": 5.805141830292018e-05, + "loss": 0.5868237495422364, + "mean_token_accuracy": 0.8236475840210915, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6527906462124415, + "eval_loss": 0.5967546105384827, + "eval_mean_token_accuracy": 0.8212518399102348, + "eval_num_tokens": 760708.0, + "eval_runtime": 110.5069, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 280 + }, + { + "entropy": 0.6342157602310181, + "epoch": 0.7361963190184049, + "grad_norm": 0.4901776611804962, + "learning_rate": 5.574059258291945e-05, + "loss": 0.5924944877624512, + "mean_token_accuracy": 0.8226904451847077, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6531158692496164, + "eval_loss": 0.589684784412384, + "eval_mean_token_accuracy": 0.8269473876271929, + "eval_num_tokens": 816117.0, + "eval_runtime": 110.5437, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 300 + }, + { + "entropy": 0.6304035305976867, + "epoch": 0.7852760736196319, + "grad_norm": 0.5012097954750061, + "learning_rate": 5.327688135467081e-05, + "loss": 0.5837351322174072, + "mean_token_accuracy": 0.8231772750616073, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6399370544297355, + "eval_loss": 0.587123692035675, + "eval_mean_token_accuracy": 0.8258680891990662, + "eval_num_tokens": 869286.0, + "eval_runtime": 110.5025, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 320 + }, + { + "entropy": 0.6135720565915108, + "epoch": 0.8343558282208589, + "grad_norm": 0.4748188555240631, + "learning_rate": 5.0678326930861606e-05, + "loss": 0.5775577068328858, + "mean_token_accuracy": 0.8291384443640709, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6397319528034755, + "eval_loss": 0.5828214287757874, + "eval_mean_token_accuracy": 0.8270208791324071, + "eval_num_tokens": 922579.0, + "eval_runtime": 110.5417, + "eval_samples_per_second": 12.629, + "eval_steps_per_second": 1.583, + "step": 340 + }, + { + "entropy": 0.6191160723567009, + "epoch": 0.8834355828220859, + "grad_norm": 0.6061940789222717, + "learning_rate": 4.796395911129629e-05, + "loss": 0.5716292858123779, + "mean_token_accuracy": 0.8298171013593674, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.6124279757908412, + "eval_loss": 0.5765473246574402, + "eval_mean_token_accuracy": 0.8290438723564147, + "eval_num_tokens": 977707.0, + "eval_runtime": 110.5552, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 360 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.01436021377024e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..783c5c54f525c00931083037996999abf4e680fe --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/trainer_state.json @@ -0,0 +1,433 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9325153374233128, + "eval_steps": 20, + "global_step": 380, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + }, + { + "entropy": 0.6266187071800232, + "epoch": 0.6871165644171779, + "grad_norm": 0.5539442300796509, + "learning_rate": 5.805141830292018e-05, + "loss": 0.5868237495422364, + "mean_token_accuracy": 0.8236475840210915, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6527906462124415, + "eval_loss": 0.5967546105384827, + "eval_mean_token_accuracy": 0.8212518399102348, + "eval_num_tokens": 760708.0, + "eval_runtime": 110.5069, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 280 + }, + { + "entropy": 0.6342157602310181, + "epoch": 0.7361963190184049, + "grad_norm": 0.4901776611804962, + "learning_rate": 5.574059258291945e-05, + "loss": 0.5924944877624512, + "mean_token_accuracy": 0.8226904451847077, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6531158692496164, + "eval_loss": 0.589684784412384, + "eval_mean_token_accuracy": 0.8269473876271929, + "eval_num_tokens": 816117.0, + "eval_runtime": 110.5437, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 300 + }, + { + "entropy": 0.6304035305976867, + "epoch": 0.7852760736196319, + "grad_norm": 0.5012097954750061, + "learning_rate": 5.327688135467081e-05, + "loss": 0.5837351322174072, + "mean_token_accuracy": 0.8231772750616073, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6399370544297355, + "eval_loss": 0.587123692035675, + "eval_mean_token_accuracy": 0.8258680891990662, + "eval_num_tokens": 869286.0, + "eval_runtime": 110.5025, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 320 + }, + { + "entropy": 0.6135720565915108, + "epoch": 0.8343558282208589, + "grad_norm": 0.4748188555240631, + "learning_rate": 5.0678326930861606e-05, + "loss": 0.5775577068328858, + "mean_token_accuracy": 0.8291384443640709, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6397319528034755, + "eval_loss": 0.5828214287757874, + "eval_mean_token_accuracy": 0.8270208791324071, + "eval_num_tokens": 922579.0, + "eval_runtime": 110.5417, + "eval_samples_per_second": 12.629, + "eval_steps_per_second": 1.583, + "step": 340 + }, + { + "entropy": 0.6191160723567009, + "epoch": 0.8834355828220859, + "grad_norm": 0.6061940789222717, + "learning_rate": 4.796395911129629e-05, + "loss": 0.5716292858123779, + "mean_token_accuracy": 0.8298171013593674, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.6124279757908412, + "eval_loss": 0.5765473246574402, + "eval_mean_token_accuracy": 0.8290438723564147, + "eval_num_tokens": 977707.0, + "eval_runtime": 110.5552, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 360 + }, + { + "entropy": 0.625142814218998, + "epoch": 0.9325153374233128, + "grad_norm": 0.5580645799636841, + "learning_rate": 4.51536558233828e-05, + "loss": 0.5804174423217774, + "mean_token_accuracy": 0.826257199048996, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.6048034725870405, + "eval_loss": 0.5730924010276794, + "eval_mean_token_accuracy": 0.8302785134315491, + "eval_num_tokens": 1028058.0, + "eval_runtime": 110.599, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 380 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.12009338662912e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..53258e99fc45263421bc3d8ebf9aa0d2991dde4b --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/trainer_state.json @@ -0,0 +1,76 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.09815950920245399, + "eval_steps": 20, + "global_step": 40, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.3358270824448e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f73970b2e1337b2146ab433e468336381488d3fb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/trainer_state.json @@ -0,0 +1,454 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9815950920245399, + "eval_steps": 20, + "global_step": 400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + }, + { + "entropy": 0.6266187071800232, + "epoch": 0.6871165644171779, + "grad_norm": 0.5539442300796509, + "learning_rate": 5.805141830292018e-05, + "loss": 0.5868237495422364, + "mean_token_accuracy": 0.8236475840210915, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6527906462124415, + "eval_loss": 0.5967546105384827, + "eval_mean_token_accuracy": 0.8212518399102348, + "eval_num_tokens": 760708.0, + "eval_runtime": 110.5069, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 280 + }, + { + "entropy": 0.6342157602310181, + "epoch": 0.7361963190184049, + "grad_norm": 0.4901776611804962, + "learning_rate": 5.574059258291945e-05, + "loss": 0.5924944877624512, + "mean_token_accuracy": 0.8226904451847077, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6531158692496164, + "eval_loss": 0.589684784412384, + "eval_mean_token_accuracy": 0.8269473876271929, + "eval_num_tokens": 816117.0, + "eval_runtime": 110.5437, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 300 + }, + { + "entropy": 0.6304035305976867, + "epoch": 0.7852760736196319, + "grad_norm": 0.5012097954750061, + "learning_rate": 5.327688135467081e-05, + "loss": 0.5837351322174072, + "mean_token_accuracy": 0.8231772750616073, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6399370544297355, + "eval_loss": 0.587123692035675, + "eval_mean_token_accuracy": 0.8258680891990662, + "eval_num_tokens": 869286.0, + "eval_runtime": 110.5025, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 320 + }, + { + "entropy": 0.6135720565915108, + "epoch": 0.8343558282208589, + "grad_norm": 0.4748188555240631, + "learning_rate": 5.0678326930861606e-05, + "loss": 0.5775577068328858, + "mean_token_accuracy": 0.8291384443640709, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6397319528034755, + "eval_loss": 0.5828214287757874, + "eval_mean_token_accuracy": 0.8270208791324071, + "eval_num_tokens": 922579.0, + "eval_runtime": 110.5417, + "eval_samples_per_second": 12.629, + "eval_steps_per_second": 1.583, + "step": 340 + }, + { + "entropy": 0.6191160723567009, + "epoch": 0.8834355828220859, + "grad_norm": 0.6061940789222717, + "learning_rate": 4.796395911129629e-05, + "loss": 0.5716292858123779, + "mean_token_accuracy": 0.8298171013593674, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.6124279757908412, + "eval_loss": 0.5765473246574402, + "eval_mean_token_accuracy": 0.8290438723564147, + "eval_num_tokens": 977707.0, + "eval_runtime": 110.5552, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 360 + }, + { + "entropy": 0.625142814218998, + "epoch": 0.9325153374233128, + "grad_norm": 0.5580645799636841, + "learning_rate": 4.51536558233828e-05, + "loss": 0.5804174423217774, + "mean_token_accuracy": 0.826257199048996, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.6048034725870405, + "eval_loss": 0.5730924010276794, + "eval_mean_token_accuracy": 0.8302785134315491, + "eval_num_tokens": 1028058.0, + "eval_runtime": 110.599, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 380 + }, + { + "entropy": 0.6035548150539398, + "epoch": 0.9815950920245399, + "grad_norm": 0.4648301303386688, + "learning_rate": 4.226799755158921e-05, + "loss": 0.5652025222778321, + "mean_token_accuracy": 0.8312336325645446, + "num_tokens": 1083860.0, + "step": 400 + }, + { + "epoch": 0.9815950920245399, + "eval_entropy": 0.5906624158791133, + "eval_loss": 0.5693612098693848, + "eval_mean_token_accuracy": 0.830716917855399, + "eval_num_tokens": 1083860.0, + "eval_runtime": 110.5521, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 400 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.2348536462336e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..cafa66c5350179bc96e046e6db1ea26c8a15ad0c --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/trainer_state.json @@ -0,0 +1,475 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0294478527607362, + "eval_steps": 20, + "global_step": 420, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + }, + { + "entropy": 0.6266187071800232, + "epoch": 0.6871165644171779, + "grad_norm": 0.5539442300796509, + "learning_rate": 5.805141830292018e-05, + "loss": 0.5868237495422364, + "mean_token_accuracy": 0.8236475840210915, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6527906462124415, + "eval_loss": 0.5967546105384827, + "eval_mean_token_accuracy": 0.8212518399102348, + "eval_num_tokens": 760708.0, + "eval_runtime": 110.5069, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 280 + }, + { + "entropy": 0.6342157602310181, + "epoch": 0.7361963190184049, + "grad_norm": 0.4901776611804962, + "learning_rate": 5.574059258291945e-05, + "loss": 0.5924944877624512, + "mean_token_accuracy": 0.8226904451847077, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6531158692496164, + "eval_loss": 0.589684784412384, + "eval_mean_token_accuracy": 0.8269473876271929, + "eval_num_tokens": 816117.0, + "eval_runtime": 110.5437, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 300 + }, + { + "entropy": 0.6304035305976867, + "epoch": 0.7852760736196319, + "grad_norm": 0.5012097954750061, + "learning_rate": 5.327688135467081e-05, + "loss": 0.5837351322174072, + "mean_token_accuracy": 0.8231772750616073, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6399370544297355, + "eval_loss": 0.587123692035675, + "eval_mean_token_accuracy": 0.8258680891990662, + "eval_num_tokens": 869286.0, + "eval_runtime": 110.5025, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 320 + }, + { + "entropy": 0.6135720565915108, + "epoch": 0.8343558282208589, + "grad_norm": 0.4748188555240631, + "learning_rate": 5.0678326930861606e-05, + "loss": 0.5775577068328858, + "mean_token_accuracy": 0.8291384443640709, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6397319528034755, + "eval_loss": 0.5828214287757874, + "eval_mean_token_accuracy": 0.8270208791324071, + "eval_num_tokens": 922579.0, + "eval_runtime": 110.5417, + "eval_samples_per_second": 12.629, + "eval_steps_per_second": 1.583, + "step": 340 + }, + { + "entropy": 0.6191160723567009, + "epoch": 0.8834355828220859, + "grad_norm": 0.6061940789222717, + "learning_rate": 4.796395911129629e-05, + "loss": 0.5716292858123779, + "mean_token_accuracy": 0.8298171013593674, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.6124279757908412, + "eval_loss": 0.5765473246574402, + "eval_mean_token_accuracy": 0.8290438723564147, + "eval_num_tokens": 977707.0, + "eval_runtime": 110.5552, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 360 + }, + { + "entropy": 0.625142814218998, + "epoch": 0.9325153374233128, + "grad_norm": 0.5580645799636841, + "learning_rate": 4.51536558233828e-05, + "loss": 0.5804174423217774, + "mean_token_accuracy": 0.826257199048996, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.6048034725870405, + "eval_loss": 0.5730924010276794, + "eval_mean_token_accuracy": 0.8302785134315491, + "eval_num_tokens": 1028058.0, + "eval_runtime": 110.599, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 380 + }, + { + "entropy": 0.6035548150539398, + "epoch": 0.9815950920245399, + "grad_norm": 0.4648301303386688, + "learning_rate": 4.226799755158921e-05, + "loss": 0.5652025222778321, + "mean_token_accuracy": 0.8312336325645446, + "num_tokens": 1083860.0, + "step": 400 + }, + { + "epoch": 0.9815950920245399, + "eval_entropy": 0.5906624158791133, + "eval_loss": 0.5693612098693848, + "eval_mean_token_accuracy": 0.830716917855399, + "eval_num_tokens": 1083860.0, + "eval_runtime": 110.5521, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 400 + }, + { + "entropy": 0.5868063202271094, + "epoch": 1.0294478527607362, + "grad_norm": 0.4341222643852234, + "learning_rate": 3.932811662191691e-05, + "loss": 0.5437146663665772, + "mean_token_accuracy": 0.836261218939072, + "num_tokens": 1136522.0, + "step": 420 + }, + { + "epoch": 1.0294478527607362, + "eval_entropy": 0.5754020828860147, + "eval_loss": 0.5691646933555603, + "eval_mean_token_accuracy": 0.8311370512417384, + "eval_num_tokens": 1136522.0, + "eval_runtime": 110.5532, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 420 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.33849503109376e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..db94016604470d798f26494f593fce66ea403cc3 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/trainer_state.json @@ -0,0 +1,496 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0785276073619632, + "eval_steps": 20, + "global_step": 440, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + }, + { + "entropy": 0.6266187071800232, + "epoch": 0.6871165644171779, + "grad_norm": 0.5539442300796509, + "learning_rate": 5.805141830292018e-05, + "loss": 0.5868237495422364, + "mean_token_accuracy": 0.8236475840210915, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6527906462124415, + "eval_loss": 0.5967546105384827, + "eval_mean_token_accuracy": 0.8212518399102348, + "eval_num_tokens": 760708.0, + "eval_runtime": 110.5069, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 280 + }, + { + "entropy": 0.6342157602310181, + "epoch": 0.7361963190184049, + "grad_norm": 0.4901776611804962, + "learning_rate": 5.574059258291945e-05, + "loss": 0.5924944877624512, + "mean_token_accuracy": 0.8226904451847077, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6531158692496164, + "eval_loss": 0.589684784412384, + "eval_mean_token_accuracy": 0.8269473876271929, + "eval_num_tokens": 816117.0, + "eval_runtime": 110.5437, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 300 + }, + { + "entropy": 0.6304035305976867, + "epoch": 0.7852760736196319, + "grad_norm": 0.5012097954750061, + "learning_rate": 5.327688135467081e-05, + "loss": 0.5837351322174072, + "mean_token_accuracy": 0.8231772750616073, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6399370544297355, + "eval_loss": 0.587123692035675, + "eval_mean_token_accuracy": 0.8258680891990662, + "eval_num_tokens": 869286.0, + "eval_runtime": 110.5025, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 320 + }, + { + "entropy": 0.6135720565915108, + "epoch": 0.8343558282208589, + "grad_norm": 0.4748188555240631, + "learning_rate": 5.0678326930861606e-05, + "loss": 0.5775577068328858, + "mean_token_accuracy": 0.8291384443640709, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6397319528034755, + "eval_loss": 0.5828214287757874, + "eval_mean_token_accuracy": 0.8270208791324071, + "eval_num_tokens": 922579.0, + "eval_runtime": 110.5417, + "eval_samples_per_second": 12.629, + "eval_steps_per_second": 1.583, + "step": 340 + }, + { + "entropy": 0.6191160723567009, + "epoch": 0.8834355828220859, + "grad_norm": 0.6061940789222717, + "learning_rate": 4.796395911129629e-05, + "loss": 0.5716292858123779, + "mean_token_accuracy": 0.8298171013593674, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.6124279757908412, + "eval_loss": 0.5765473246574402, + "eval_mean_token_accuracy": 0.8290438723564147, + "eval_num_tokens": 977707.0, + "eval_runtime": 110.5552, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 360 + }, + { + "entropy": 0.625142814218998, + "epoch": 0.9325153374233128, + "grad_norm": 0.5580645799636841, + "learning_rate": 4.51536558233828e-05, + "loss": 0.5804174423217774, + "mean_token_accuracy": 0.826257199048996, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.6048034725870405, + "eval_loss": 0.5730924010276794, + "eval_mean_token_accuracy": 0.8302785134315491, + "eval_num_tokens": 1028058.0, + "eval_runtime": 110.599, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 380 + }, + { + "entropy": 0.6035548150539398, + "epoch": 0.9815950920245399, + "grad_norm": 0.4648301303386688, + "learning_rate": 4.226799755158921e-05, + "loss": 0.5652025222778321, + "mean_token_accuracy": 0.8312336325645446, + "num_tokens": 1083860.0, + "step": 400 + }, + { + "epoch": 0.9815950920245399, + "eval_entropy": 0.5906624158791133, + "eval_loss": 0.5693612098693848, + "eval_mean_token_accuracy": 0.830716917855399, + "eval_num_tokens": 1083860.0, + "eval_runtime": 110.5521, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 400 + }, + { + "entropy": 0.5868063202271094, + "epoch": 1.0294478527607362, + "grad_norm": 0.4341222643852234, + "learning_rate": 3.932811662191691e-05, + "loss": 0.5437146663665772, + "mean_token_accuracy": 0.836261218939072, + "num_tokens": 1136522.0, + "step": 420 + }, + { + "epoch": 1.0294478527607362, + "eval_entropy": 0.5754020828860147, + "eval_loss": 0.5691646933555603, + "eval_mean_token_accuracy": 0.8311370512417384, + "eval_num_tokens": 1136522.0, + "eval_runtime": 110.5532, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 420 + }, + { + "entropy": 0.5655149511992932, + "epoch": 1.0785276073619632, + "grad_norm": 0.45058947801589966, + "learning_rate": 3.635554244511366e-05, + "loss": 0.530416202545166, + "mean_token_accuracy": 0.8381419599056243, + "num_tokens": 1196771.0, + "step": 440 + }, + { + "epoch": 1.0785276073619632, + "eval_entropy": 0.5802437552383968, + "eval_loss": 0.5651134252548218, + "eval_mean_token_accuracy": 0.8319863060542515, + "eval_num_tokens": 1196771.0, + "eval_runtime": 110.5729, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 440 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.46072999207168e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b67b6433d6d3ccf1452ea98087435e08862295d0 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/trainer_state.json @@ -0,0 +1,517 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.1276073619631901, + "eval_steps": 20, + "global_step": 460, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + }, + { + "entropy": 0.6266187071800232, + "epoch": 0.6871165644171779, + "grad_norm": 0.5539442300796509, + "learning_rate": 5.805141830292018e-05, + "loss": 0.5868237495422364, + "mean_token_accuracy": 0.8236475840210915, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6527906462124415, + "eval_loss": 0.5967546105384827, + "eval_mean_token_accuracy": 0.8212518399102348, + "eval_num_tokens": 760708.0, + "eval_runtime": 110.5069, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 280 + }, + { + "entropy": 0.6342157602310181, + "epoch": 0.7361963190184049, + "grad_norm": 0.4901776611804962, + "learning_rate": 5.574059258291945e-05, + "loss": 0.5924944877624512, + "mean_token_accuracy": 0.8226904451847077, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6531158692496164, + "eval_loss": 0.589684784412384, + "eval_mean_token_accuracy": 0.8269473876271929, + "eval_num_tokens": 816117.0, + "eval_runtime": 110.5437, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 300 + }, + { + "entropy": 0.6304035305976867, + "epoch": 0.7852760736196319, + "grad_norm": 0.5012097954750061, + "learning_rate": 5.327688135467081e-05, + "loss": 0.5837351322174072, + "mean_token_accuracy": 0.8231772750616073, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6399370544297355, + "eval_loss": 0.587123692035675, + "eval_mean_token_accuracy": 0.8258680891990662, + "eval_num_tokens": 869286.0, + "eval_runtime": 110.5025, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 320 + }, + { + "entropy": 0.6135720565915108, + "epoch": 0.8343558282208589, + "grad_norm": 0.4748188555240631, + "learning_rate": 5.0678326930861606e-05, + "loss": 0.5775577068328858, + "mean_token_accuracy": 0.8291384443640709, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6397319528034755, + "eval_loss": 0.5828214287757874, + "eval_mean_token_accuracy": 0.8270208791324071, + "eval_num_tokens": 922579.0, + "eval_runtime": 110.5417, + "eval_samples_per_second": 12.629, + "eval_steps_per_second": 1.583, + "step": 340 + }, + { + "entropy": 0.6191160723567009, + "epoch": 0.8834355828220859, + "grad_norm": 0.6061940789222717, + "learning_rate": 4.796395911129629e-05, + "loss": 0.5716292858123779, + "mean_token_accuracy": 0.8298171013593674, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.6124279757908412, + "eval_loss": 0.5765473246574402, + "eval_mean_token_accuracy": 0.8290438723564147, + "eval_num_tokens": 977707.0, + "eval_runtime": 110.5552, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 360 + }, + { + "entropy": 0.625142814218998, + "epoch": 0.9325153374233128, + "grad_norm": 0.5580645799636841, + "learning_rate": 4.51536558233828e-05, + "loss": 0.5804174423217774, + "mean_token_accuracy": 0.826257199048996, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.6048034725870405, + "eval_loss": 0.5730924010276794, + "eval_mean_token_accuracy": 0.8302785134315491, + "eval_num_tokens": 1028058.0, + "eval_runtime": 110.599, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 380 + }, + { + "entropy": 0.6035548150539398, + "epoch": 0.9815950920245399, + "grad_norm": 0.4648301303386688, + "learning_rate": 4.226799755158921e-05, + "loss": 0.5652025222778321, + "mean_token_accuracy": 0.8312336325645446, + "num_tokens": 1083860.0, + "step": 400 + }, + { + "epoch": 0.9815950920245399, + "eval_entropy": 0.5906624158791133, + "eval_loss": 0.5693612098693848, + "eval_mean_token_accuracy": 0.830716917855399, + "eval_num_tokens": 1083860.0, + "eval_runtime": 110.5521, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 400 + }, + { + "entropy": 0.5868063202271094, + "epoch": 1.0294478527607362, + "grad_norm": 0.4341222643852234, + "learning_rate": 3.932811662191691e-05, + "loss": 0.5437146663665772, + "mean_token_accuracy": 0.836261218939072, + "num_tokens": 1136522.0, + "step": 420 + }, + { + "epoch": 1.0294478527607362, + "eval_entropy": 0.5754020828860147, + "eval_loss": 0.5691646933555603, + "eval_mean_token_accuracy": 0.8311370512417384, + "eval_num_tokens": 1136522.0, + "eval_runtime": 110.5532, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 420 + }, + { + "entropy": 0.5655149511992932, + "epoch": 1.0785276073619632, + "grad_norm": 0.45058947801589966, + "learning_rate": 3.635554244511366e-05, + "loss": 0.530416202545166, + "mean_token_accuracy": 0.8381419599056243, + "num_tokens": 1196771.0, + "step": 440 + }, + { + "epoch": 1.0785276073619632, + "eval_entropy": 0.5802437552383968, + "eval_loss": 0.5651134252548218, + "eval_mean_token_accuracy": 0.8319863060542515, + "eval_num_tokens": 1196771.0, + "eval_runtime": 110.5729, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 440 + }, + { + "entropy": 0.5608904980123043, + "epoch": 1.1276073619631901, + "grad_norm": 0.5339303612709045, + "learning_rate": 3.337204385194556e-05, + "loss": 0.5203728675842285, + "mean_token_accuracy": 0.840325091779232, + "num_tokens": 1252972.0, + "step": 460 + }, + { + "epoch": 1.1276073619631901, + "eval_entropy": 0.5674476792131151, + "eval_loss": 0.5644519329071045, + "eval_mean_token_accuracy": 0.8326545231682914, + "eval_num_tokens": 1252972.0, + "eval_runtime": 110.5291, + "eval_samples_per_second": 12.63, + "eval_steps_per_second": 1.583, + "step": 460 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.5787933447808e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..88cdcaf53816707718829e2ebdbb54392adbf956 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/trainer_state.json @@ -0,0 +1,538 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.1766871165644173, + "eval_steps": 20, + "global_step": 480, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + }, + { + "entropy": 0.6266187071800232, + "epoch": 0.6871165644171779, + "grad_norm": 0.5539442300796509, + "learning_rate": 5.805141830292018e-05, + "loss": 0.5868237495422364, + "mean_token_accuracy": 0.8236475840210915, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6527906462124415, + "eval_loss": 0.5967546105384827, + "eval_mean_token_accuracy": 0.8212518399102348, + "eval_num_tokens": 760708.0, + "eval_runtime": 110.5069, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 280 + }, + { + "entropy": 0.6342157602310181, + "epoch": 0.7361963190184049, + "grad_norm": 0.4901776611804962, + "learning_rate": 5.574059258291945e-05, + "loss": 0.5924944877624512, + "mean_token_accuracy": 0.8226904451847077, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6531158692496164, + "eval_loss": 0.589684784412384, + "eval_mean_token_accuracy": 0.8269473876271929, + "eval_num_tokens": 816117.0, + "eval_runtime": 110.5437, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 300 + }, + { + "entropy": 0.6304035305976867, + "epoch": 0.7852760736196319, + "grad_norm": 0.5012097954750061, + "learning_rate": 5.327688135467081e-05, + "loss": 0.5837351322174072, + "mean_token_accuracy": 0.8231772750616073, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6399370544297355, + "eval_loss": 0.587123692035675, + "eval_mean_token_accuracy": 0.8258680891990662, + "eval_num_tokens": 869286.0, + "eval_runtime": 110.5025, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 320 + }, + { + "entropy": 0.6135720565915108, + "epoch": 0.8343558282208589, + "grad_norm": 0.4748188555240631, + "learning_rate": 5.0678326930861606e-05, + "loss": 0.5775577068328858, + "mean_token_accuracy": 0.8291384443640709, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6397319528034755, + "eval_loss": 0.5828214287757874, + "eval_mean_token_accuracy": 0.8270208791324071, + "eval_num_tokens": 922579.0, + "eval_runtime": 110.5417, + "eval_samples_per_second": 12.629, + "eval_steps_per_second": 1.583, + "step": 340 + }, + { + "entropy": 0.6191160723567009, + "epoch": 0.8834355828220859, + "grad_norm": 0.6061940789222717, + "learning_rate": 4.796395911129629e-05, + "loss": 0.5716292858123779, + "mean_token_accuracy": 0.8298171013593674, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.6124279757908412, + "eval_loss": 0.5765473246574402, + "eval_mean_token_accuracy": 0.8290438723564147, + "eval_num_tokens": 977707.0, + "eval_runtime": 110.5552, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 360 + }, + { + "entropy": 0.625142814218998, + "epoch": 0.9325153374233128, + "grad_norm": 0.5580645799636841, + "learning_rate": 4.51536558233828e-05, + "loss": 0.5804174423217774, + "mean_token_accuracy": 0.826257199048996, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.6048034725870405, + "eval_loss": 0.5730924010276794, + "eval_mean_token_accuracy": 0.8302785134315491, + "eval_num_tokens": 1028058.0, + "eval_runtime": 110.599, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 380 + }, + { + "entropy": 0.6035548150539398, + "epoch": 0.9815950920245399, + "grad_norm": 0.4648301303386688, + "learning_rate": 4.226799755158921e-05, + "loss": 0.5652025222778321, + "mean_token_accuracy": 0.8312336325645446, + "num_tokens": 1083860.0, + "step": 400 + }, + { + "epoch": 0.9815950920245399, + "eval_entropy": 0.5906624158791133, + "eval_loss": 0.5693612098693848, + "eval_mean_token_accuracy": 0.830716917855399, + "eval_num_tokens": 1083860.0, + "eval_runtime": 110.5521, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 400 + }, + { + "entropy": 0.5868063202271094, + "epoch": 1.0294478527607362, + "grad_norm": 0.4341222643852234, + "learning_rate": 3.932811662191691e-05, + "loss": 0.5437146663665772, + "mean_token_accuracy": 0.836261218939072, + "num_tokens": 1136522.0, + "step": 420 + }, + { + "epoch": 1.0294478527607362, + "eval_entropy": 0.5754020828860147, + "eval_loss": 0.5691646933555603, + "eval_mean_token_accuracy": 0.8311370512417384, + "eval_num_tokens": 1136522.0, + "eval_runtime": 110.5532, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 420 + }, + { + "entropy": 0.5655149511992932, + "epoch": 1.0785276073619632, + "grad_norm": 0.45058947801589966, + "learning_rate": 3.635554244511366e-05, + "loss": 0.530416202545166, + "mean_token_accuracy": 0.8381419599056243, + "num_tokens": 1196771.0, + "step": 440 + }, + { + "epoch": 1.0785276073619632, + "eval_entropy": 0.5802437552383968, + "eval_loss": 0.5651134252548218, + "eval_mean_token_accuracy": 0.8319863060542515, + "eval_num_tokens": 1196771.0, + "eval_runtime": 110.5729, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 440 + }, + { + "entropy": 0.5608904980123043, + "epoch": 1.1276073619631901, + "grad_norm": 0.5339303612709045, + "learning_rate": 3.337204385194556e-05, + "loss": 0.5203728675842285, + "mean_token_accuracy": 0.840325091779232, + "num_tokens": 1252972.0, + "step": 460 + }, + { + "epoch": 1.1276073619631901, + "eval_entropy": 0.5674476792131151, + "eval_loss": 0.5644519329071045, + "eval_mean_token_accuracy": 0.8326545231682914, + "eval_num_tokens": 1252972.0, + "eval_runtime": 110.5291, + "eval_samples_per_second": 12.63, + "eval_steps_per_second": 1.583, + "step": 460 + }, + { + "entropy": 0.5624608777463436, + "epoch": 1.1766871165644173, + "grad_norm": 0.4759756028652191, + "learning_rate": 3.039946967514229e-05, + "loss": 0.5192633152008057, + "mean_token_accuracy": 0.8422753617167473, + "num_tokens": 1304559.0, + "step": 480 + }, + { + "epoch": 1.1766871165644173, + "eval_entropy": 0.5709343756948199, + "eval_loss": 0.5619797110557556, + "eval_mean_token_accuracy": 0.8334083775111607, + "eval_num_tokens": 1304559.0, + "eval_runtime": 110.5431, + "eval_samples_per_second": 12.629, + "eval_steps_per_second": 1.583, + "step": 480 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.6827074228864e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3938a913512400eec1decacf9251d82972438496 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/trainer_state.json @@ -0,0 +1,559 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.2257668711656442, + "eval_steps": 20, + "global_step": 500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + }, + { + "entropy": 0.7375246778130531, + "epoch": 0.24539877300613497, + "grad_norm": 0.9057678580284119, + "learning_rate": 6.96353379709049e-05, + "loss": 0.702963924407959, + "mean_token_accuracy": 0.797600856423378, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7367377580915179, + "eval_loss": 0.6845751404762268, + "eval_mean_token_accuracy": 0.8011933132580349, + "eval_num_tokens": 267691.0, + "eval_runtime": 110.6372, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 100 + }, + { + "entropy": 0.7158783033490181, + "epoch": 0.294478527607362, + "grad_norm": 0.4919581413269043, + "learning_rate": 6.929132379744316e-05, + "loss": 0.6752747058868408, + "mean_token_accuracy": 0.8034310981631279, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.7097335999352591, + "eval_loss": 0.6642724275588989, + "eval_mean_token_accuracy": 0.8056706874711173, + "eval_num_tokens": 323653.0, + "eval_runtime": 110.6492, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 120 + }, + { + "entropy": 0.6817873455584049, + "epoch": 0.34355828220858897, + "grad_norm": 0.4943574368953705, + "learning_rate": 6.86951890564815e-05, + "loss": 0.6389606952667236, + "mean_token_accuracy": 0.8110431596636772, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6846639224461146, + "eval_loss": 0.6477498412132263, + "eval_mean_token_accuracy": 0.8120035709653581, + "eval_num_tokens": 382063.0, + "eval_runtime": 110.6491, + "eval_samples_per_second": 12.616, + "eval_steps_per_second": 1.582, + "step": 140 + }, + { + "entropy": 0.6678083419799805, + "epoch": 0.39263803680981596, + "grad_norm": 0.4057321548461914, + "learning_rate": 6.785129937710703e-05, + "loss": 0.6308637619018554, + "mean_token_accuracy": 0.816147631406784, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6869715414728437, + "eval_loss": 0.6331504583358765, + "eval_mean_token_accuracy": 0.8155754739897592, + "eval_num_tokens": 440825.0, + "eval_runtime": 110.673, + "eval_samples_per_second": 12.614, + "eval_steps_per_second": 1.581, + "step": 160 + }, + { + "entropy": 0.6723565131425857, + "epoch": 0.44171779141104295, + "grad_norm": 0.6217795610427856, + "learning_rate": 6.676583475366906e-05, + "loss": 0.62900390625, + "mean_token_accuracy": 0.8144089907407761, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6551390927178519, + "eval_loss": 0.6275219917297363, + "eval_mean_token_accuracy": 0.8162073765482222, + "eval_num_tokens": 494124.0, + "eval_runtime": 110.6478, + "eval_samples_per_second": 12.617, + "eval_steps_per_second": 1.582, + "step": 180 + }, + { + "entropy": 0.6538991808891297, + "epoch": 0.49079754601226994, + "grad_norm": 0.5870281457901001, + "learning_rate": 6.544674428828742e-05, + "loss": 0.6221371173858643, + "mean_token_accuracy": 0.81730937063694, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6728082823753357, + "eval_loss": 0.6223575472831726, + "eval_mean_token_accuracy": 0.8153460209710257, + "eval_num_tokens": 542703.0, + "eval_runtime": 110.6384, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 200 + }, + { + "entropy": 0.6546640872955323, + "epoch": 0.5398773006134969, + "grad_norm": 1.105159044265747, + "learning_rate": 6.390368797778584e-05, + "loss": 0.6159460067749023, + "mean_token_accuracy": 0.8187728151679039, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6527067743028914, + "eval_loss": 0.6144458055496216, + "eval_mean_token_accuracy": 0.8200722203935895, + "eval_num_tokens": 597461.0, + "eval_runtime": 110.6247, + "eval_samples_per_second": 12.619, + "eval_steps_per_second": 1.582, + "step": 220 + }, + { + "entropy": 0.6596392050385476, + "epoch": 0.588957055214724, + "grad_norm": 0.5103256106376648, + "learning_rate": 6.214796597135754e-05, + "loss": 0.6109931468963623, + "mean_token_accuracy": 0.8179920643568039, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.671152674811227, + "eval_loss": 0.6105258464813232, + "eval_mean_token_accuracy": 0.8179612030301775, + "eval_num_tokens": 651354.0, + "eval_runtime": 110.5976, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 240 + }, + { + "entropy": 0.625977823138237, + "epoch": 0.6380368098159509, + "grad_norm": 0.5184243321418762, + "learning_rate": 6.019243581702554e-05, + "loss": 0.5913448333740234, + "mean_token_accuracy": 0.8233017310500145, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6445648077556065, + "eval_loss": 0.601780891418457, + "eval_mean_token_accuracy": 0.823449080671583, + "eval_num_tokens": 707219.0, + "eval_runtime": 110.576, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 260 + }, + { + "entropy": 0.6266187071800232, + "epoch": 0.6871165644171779, + "grad_norm": 0.5539442300796509, + "learning_rate": 5.805141830292018e-05, + "loss": 0.5868237495422364, + "mean_token_accuracy": 0.8236475840210915, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6527906462124415, + "eval_loss": 0.5967546105384827, + "eval_mean_token_accuracy": 0.8212518399102348, + "eval_num_tokens": 760708.0, + "eval_runtime": 110.5069, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 280 + }, + { + "entropy": 0.6342157602310181, + "epoch": 0.7361963190184049, + "grad_norm": 0.4901776611804962, + "learning_rate": 5.574059258291945e-05, + "loss": 0.5924944877624512, + "mean_token_accuracy": 0.8226904451847077, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6531158692496164, + "eval_loss": 0.589684784412384, + "eval_mean_token_accuracy": 0.8269473876271929, + "eval_num_tokens": 816117.0, + "eval_runtime": 110.5437, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 300 + }, + { + "entropy": 0.6304035305976867, + "epoch": 0.7852760736196319, + "grad_norm": 0.5012097954750061, + "learning_rate": 5.327688135467081e-05, + "loss": 0.5837351322174072, + "mean_token_accuracy": 0.8231772750616073, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6399370544297355, + "eval_loss": 0.587123692035675, + "eval_mean_token_accuracy": 0.8258680891990662, + "eval_num_tokens": 869286.0, + "eval_runtime": 110.5025, + "eval_samples_per_second": 12.633, + "eval_steps_per_second": 1.584, + "step": 320 + }, + { + "entropy": 0.6135720565915108, + "epoch": 0.8343558282208589, + "grad_norm": 0.4748188555240631, + "learning_rate": 5.0678326930861606e-05, + "loss": 0.5775577068328858, + "mean_token_accuracy": 0.8291384443640709, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6397319528034755, + "eval_loss": 0.5828214287757874, + "eval_mean_token_accuracy": 0.8270208791324071, + "eval_num_tokens": 922579.0, + "eval_runtime": 110.5417, + "eval_samples_per_second": 12.629, + "eval_steps_per_second": 1.583, + "step": 340 + }, + { + "entropy": 0.6191160723567009, + "epoch": 0.8834355828220859, + "grad_norm": 0.6061940789222717, + "learning_rate": 4.796395911129629e-05, + "loss": 0.5716292858123779, + "mean_token_accuracy": 0.8298171013593674, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.6124279757908412, + "eval_loss": 0.5765473246574402, + "eval_mean_token_accuracy": 0.8290438723564147, + "eval_num_tokens": 977707.0, + "eval_runtime": 110.5552, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 360 + }, + { + "entropy": 0.625142814218998, + "epoch": 0.9325153374233128, + "grad_norm": 0.5580645799636841, + "learning_rate": 4.51536558233828e-05, + "loss": 0.5804174423217774, + "mean_token_accuracy": 0.826257199048996, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.6048034725870405, + "eval_loss": 0.5730924010276794, + "eval_mean_token_accuracy": 0.8302785134315491, + "eval_num_tokens": 1028058.0, + "eval_runtime": 110.599, + "eval_samples_per_second": 12.622, + "eval_steps_per_second": 1.582, + "step": 380 + }, + { + "entropy": 0.6035548150539398, + "epoch": 0.9815950920245399, + "grad_norm": 0.4648301303386688, + "learning_rate": 4.226799755158921e-05, + "loss": 0.5652025222778321, + "mean_token_accuracy": 0.8312336325645446, + "num_tokens": 1083860.0, + "step": 400 + }, + { + "epoch": 0.9815950920245399, + "eval_entropy": 0.5906624158791133, + "eval_loss": 0.5693612098693848, + "eval_mean_token_accuracy": 0.830716917855399, + "eval_num_tokens": 1083860.0, + "eval_runtime": 110.5521, + "eval_samples_per_second": 12.628, + "eval_steps_per_second": 1.583, + "step": 400 + }, + { + "entropy": 0.5868063202271094, + "epoch": 1.0294478527607362, + "grad_norm": 0.4341222643852234, + "learning_rate": 3.932811662191691e-05, + "loss": 0.5437146663665772, + "mean_token_accuracy": 0.836261218939072, + "num_tokens": 1136522.0, + "step": 420 + }, + { + "epoch": 1.0294478527607362, + "eval_entropy": 0.5754020828860147, + "eval_loss": 0.5691646933555603, + "eval_mean_token_accuracy": 0.8311370512417384, + "eval_num_tokens": 1136522.0, + "eval_runtime": 110.5532, + "eval_samples_per_second": 12.627, + "eval_steps_per_second": 1.583, + "step": 420 + }, + { + "entropy": 0.5655149511992932, + "epoch": 1.0785276073619632, + "grad_norm": 0.45058947801589966, + "learning_rate": 3.635554244511366e-05, + "loss": 0.530416202545166, + "mean_token_accuracy": 0.8381419599056243, + "num_tokens": 1196771.0, + "step": 440 + }, + { + "epoch": 1.0785276073619632, + "eval_entropy": 0.5802437552383968, + "eval_loss": 0.5651134252548218, + "eval_mean_token_accuracy": 0.8319863060542515, + "eval_num_tokens": 1196771.0, + "eval_runtime": 110.5729, + "eval_samples_per_second": 12.625, + "eval_steps_per_second": 1.583, + "step": 440 + }, + { + "entropy": 0.5608904980123043, + "epoch": 1.1276073619631901, + "grad_norm": 0.5339303612709045, + "learning_rate": 3.337204385194556e-05, + "loss": 0.5203728675842285, + "mean_token_accuracy": 0.840325091779232, + "num_tokens": 1252972.0, + "step": 460 + }, + { + "epoch": 1.1276073619631901, + "eval_entropy": 0.5674476792131151, + "eval_loss": 0.5644519329071045, + "eval_mean_token_accuracy": 0.8326545231682914, + "eval_num_tokens": 1252972.0, + "eval_runtime": 110.5291, + "eval_samples_per_second": 12.63, + "eval_steps_per_second": 1.583, + "step": 460 + }, + { + "entropy": 0.5624608777463436, + "epoch": 1.1766871165644173, + "grad_norm": 0.4759756028652191, + "learning_rate": 3.039946967514229e-05, + "loss": 0.5192633152008057, + "mean_token_accuracy": 0.8422753617167473, + "num_tokens": 1304559.0, + "step": 480 + }, + { + "epoch": 1.1766871165644173, + "eval_entropy": 0.5709343756948199, + "eval_loss": 0.5619797110557556, + "eval_mean_token_accuracy": 0.8334083775111607, + "eval_num_tokens": 1304559.0, + "eval_runtime": 110.5431, + "eval_samples_per_second": 12.629, + "eval_steps_per_second": 1.583, + "step": 480 + }, + { + "entropy": 0.545372461527586, + "epoch": 1.2257668711656442, + "grad_norm": 0.5064002275466919, + "learning_rate": 2.7459588745470005e-05, + "loss": 0.5095031261444092, + "mean_token_accuracy": 0.8455584317445755, + "num_tokens": 1362151.0, + "step": 500 + }, + { + "epoch": 1.2257668711656442, + "eval_entropy": 0.5721048392568315, + "eval_loss": 0.5595700740814209, + "eval_mean_token_accuracy": 0.8340616001401628, + "eval_num_tokens": 1362151.0, + "eval_runtime": 110.5259, + "eval_samples_per_second": 12.631, + "eval_steps_per_second": 1.583, + "step": 500 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.79402781534464e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..14de7e3027d223b3539abb5fc11ea7ebcb914c24 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/trainer_state.json @@ -0,0 +1,97 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.147239263803681, + "eval_steps": 20, + "global_step": 60, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.4977225658368e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e46eba37c7ef67d103f30399403dcb5dd5075042 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.08754516765392441, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..aa245616771db119282fc0963dc051d3438f85c8 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/trainer_state.json @@ -0,0 +1,118 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.19631901840490798, + "eval_steps": 20, + "global_step": 80, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8042113453149795, + "epoch": 0.049079754601226995, + "grad_norm": 0.8720105886459351, + "learning_rate": 1.6156391946879574e-05, + "loss": 1.7916599273681642, + "mean_token_accuracy": 0.6300179943442344, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.6253603546960014, + "eval_loss": 1.5587899684906006, + "eval_mean_token_accuracy": 0.6712809375354222, + "eval_num_tokens": 54280.0, + "eval_runtime": 110.7816, + "eval_samples_per_second": 12.601, + "eval_steps_per_second": 1.58, + "step": 20 + }, + { + "entropy": 1.2205769553780557, + "epoch": 0.09815950920245399, + "grad_norm": 0.6686593890190125, + "learning_rate": 3.316312031201596e-05, + "loss": 1.136343288421631, + "mean_token_accuracy": 0.7222879514098167, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.9434821823665074, + "eval_loss": 0.881726086139679, + "eval_mean_token_accuracy": 0.7615073473112923, + "eval_num_tokens": 113362.0, + "eval_runtime": 110.6353, + "eval_samples_per_second": 12.618, + "eval_steps_per_second": 1.582, + "step": 40 + }, + { + "entropy": 0.8354097574949264, + "epoch": 0.147239263803681, + "grad_norm": 0.6986415982246399, + "learning_rate": 5.016984867715235e-05, + "loss": 0.8029017448425293, + "mean_token_accuracy": 0.7783494427800178, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7871972390583584, + "eval_loss": 0.7541570067405701, + "eval_mean_token_accuracy": 0.7862869017464774, + "eval_num_tokens": 165819.0, + "eval_runtime": 110.5664, + "eval_samples_per_second": 12.626, + "eval_steps_per_second": 1.583, + "step": 60 + }, + { + "entropy": 0.7618587836623192, + "epoch": 0.19631901840490798, + "grad_norm": 0.6989718675613403, + "learning_rate": 6.717657704228874e-05, + "loss": 0.7183277606964111, + "mean_token_accuracy": 0.7918156325817108, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7207223660605294, + "eval_loss": 0.7117754817008972, + "eval_mean_token_accuracy": 0.7968860483169555, + "eval_num_tokens": 215870.0, + "eval_runtime": 110.6199, + "eval_samples_per_second": 12.62, + "eval_steps_per_second": 1.582, + "step": 80 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.6421793589248e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..096bea862b88d351b9f16bd7922aab0d576c21c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/trainer_state.json @@ -0,0 +1,496 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0785276073619632, + "eval_steps": 20, + "global_step": 440, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + }, + { + "entropy": 0.6182309970259666, + "epoch": 0.6871165644171779, + "grad_norm": 0.3791189193725586, + "learning_rate": 0.0002781205962525053, + "loss": 0.5778349876403809, + "mean_token_accuracy": 0.8246920928359032, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6430293495314462, + "eval_loss": 0.584140419960022, + "eval_mean_token_accuracy": 0.8250665313856942, + "eval_num_tokens": 760708.0, + "eval_runtime": 105.0023, + "eval_samples_per_second": 13.285, + "eval_steps_per_second": 1.667, + "step": 280 + }, + { + "entropy": 0.6252344503998757, + "epoch": 0.7361963190184049, + "grad_norm": 0.3159662187099457, + "learning_rate": 0.0002670495794561784, + "loss": 0.5845423221588135, + "mean_token_accuracy": 0.8252305090427399, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6507578223092215, + "eval_loss": 0.5797898173332214, + "eval_mean_token_accuracy": 0.8279903977257865, + "eval_num_tokens": 816117.0, + "eval_runtime": 104.8929, + "eval_samples_per_second": 13.299, + "eval_steps_per_second": 1.668, + "step": 300 + }, + { + "entropy": 0.6199106499552727, + "epoch": 0.7852760736196319, + "grad_norm": 0.33928635716438293, + "learning_rate": 0.0002552460980628558, + "loss": 0.5756847858428955, + "mean_token_accuracy": 0.8251789346337318, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6414087894984654, + "eval_loss": 0.5759831070899963, + "eval_mean_token_accuracy": 0.8279267614228385, + "eval_num_tokens": 869286.0, + "eval_runtime": 104.9796, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 320 + }, + { + "entropy": 0.6080872133374214, + "epoch": 0.8343558282208589, + "grad_norm": 0.2935579717159271, + "learning_rate": 0.00024279659162748857, + "loss": 0.56481032371521, + "mean_token_accuracy": 0.8329314738512039, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6243992117473057, + "eval_loss": 0.5680164694786072, + "eval_mean_token_accuracy": 0.8308778762817383, + "eval_num_tokens": 922579.0, + "eval_runtime": 104.9853, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 340 + }, + { + "entropy": 0.6098237752914428, + "epoch": 0.8834355828220859, + "grad_norm": 0.3914716839790344, + "learning_rate": 0.0002297922306920359, + "loss": 0.559494924545288, + "mean_token_accuracy": 0.8337442219257355, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.59963459951537, + "eval_loss": 0.5661377310752869, + "eval_mean_token_accuracy": 0.8304069665500096, + "eval_num_tokens": 977707.0, + "eval_runtime": 104.9926, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 360 + }, + { + "entropy": 0.6145547524094581, + "epoch": 0.9325153374233128, + "grad_norm": 0.3521580696105957, + "learning_rate": 0.00021632824912303506, + "loss": 0.5705258846282959, + "mean_token_accuracy": 0.829042598605156, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.5960987096173422, + "eval_loss": 0.5600873231887817, + "eval_mean_token_accuracy": 0.8332455556733268, + "eval_num_tokens": 1028058.0, + "eval_runtime": 104.8158, + "eval_samples_per_second": 13.309, + "eval_steps_per_second": 1.67, + "step": 380 + }, + { + "entropy": 0.5970950566232205, + "epoch": 0.9815950920245399, + "grad_norm": 0.28969496488571167, + "learning_rate": 0.00020250324669252877, + "loss": 0.5550286293029785, + "mean_token_accuracy": 0.8330720633268356, + "num_tokens": 1083860.0, + "step": 400 + }, + { + "epoch": 0.9815950920245399, + "eval_entropy": 0.590186082805906, + "eval_loss": 0.5563607215881348, + "eval_mean_token_accuracy": 0.833972008228302, + "eval_num_tokens": 1083860.0, + "eval_runtime": 104.9587, + "eval_samples_per_second": 13.291, + "eval_steps_per_second": 1.667, + "step": 400 + }, + { + "entropy": 0.5686816084079254, + "epoch": 1.0294478527607362, + "grad_norm": 0.30075210332870483, + "learning_rate": 0.0001884184670097092, + "loss": 0.5245012760162353, + "mean_token_accuracy": 0.8398806681999793, + "num_tokens": 1136522.0, + "step": 420 + }, + { + "epoch": 1.0294478527607362, + "eval_entropy": 0.5584452276570456, + "eval_loss": 0.557037889957428, + "eval_mean_token_accuracy": 0.8341134299550738, + "eval_num_tokens": 1136522.0, + "eval_runtime": 104.9666, + "eval_samples_per_second": 13.29, + "eval_steps_per_second": 1.667, + "step": 420 + }, + { + "entropy": 0.5519779615104199, + "epoch": 1.0785276073619632, + "grad_norm": 0.27849081158638, + "learning_rate": 0.00017417705609114545, + "loss": 0.5106734275817871, + "mean_token_accuracy": 0.8430664092302322, + "num_tokens": 1196771.0, + "step": 440 + }, + { + "epoch": 1.0785276073619632, + "eval_entropy": 0.5666922775336675, + "eval_loss": 0.5501689910888672, + "eval_mean_token_accuracy": 0.8352916158948626, + "eval_num_tokens": 1196771.0, + "eval_runtime": 104.9138, + "eval_samples_per_second": 13.297, + "eval_steps_per_second": 1.668, + "step": 440 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.4274518726140928e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9a2615cef5f8a0cbaf8546f6a8c614de704d42e3 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/trainer_state.json @@ -0,0 +1,517 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.1276073619631901, + "eval_steps": 20, + "global_step": 460, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + }, + { + "entropy": 0.6182309970259666, + "epoch": 0.6871165644171779, + "grad_norm": 0.3791189193725586, + "learning_rate": 0.0002781205962525053, + "loss": 0.5778349876403809, + "mean_token_accuracy": 0.8246920928359032, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6430293495314462, + "eval_loss": 0.584140419960022, + "eval_mean_token_accuracy": 0.8250665313856942, + "eval_num_tokens": 760708.0, + "eval_runtime": 105.0023, + "eval_samples_per_second": 13.285, + "eval_steps_per_second": 1.667, + "step": 280 + }, + { + "entropy": 0.6252344503998757, + "epoch": 0.7361963190184049, + "grad_norm": 0.3159662187099457, + "learning_rate": 0.0002670495794561784, + "loss": 0.5845423221588135, + "mean_token_accuracy": 0.8252305090427399, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6507578223092215, + "eval_loss": 0.5797898173332214, + "eval_mean_token_accuracy": 0.8279903977257865, + "eval_num_tokens": 816117.0, + "eval_runtime": 104.8929, + "eval_samples_per_second": 13.299, + "eval_steps_per_second": 1.668, + "step": 300 + }, + { + "entropy": 0.6199106499552727, + "epoch": 0.7852760736196319, + "grad_norm": 0.33928635716438293, + "learning_rate": 0.0002552460980628558, + "loss": 0.5756847858428955, + "mean_token_accuracy": 0.8251789346337318, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6414087894984654, + "eval_loss": 0.5759831070899963, + "eval_mean_token_accuracy": 0.8279267614228385, + "eval_num_tokens": 869286.0, + "eval_runtime": 104.9796, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 320 + }, + { + "entropy": 0.6080872133374214, + "epoch": 0.8343558282208589, + "grad_norm": 0.2935579717159271, + "learning_rate": 0.00024279659162748857, + "loss": 0.56481032371521, + "mean_token_accuracy": 0.8329314738512039, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6243992117473057, + "eval_loss": 0.5680164694786072, + "eval_mean_token_accuracy": 0.8308778762817383, + "eval_num_tokens": 922579.0, + "eval_runtime": 104.9853, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 340 + }, + { + "entropy": 0.6098237752914428, + "epoch": 0.8834355828220859, + "grad_norm": 0.3914716839790344, + "learning_rate": 0.0002297922306920359, + "loss": 0.559494924545288, + "mean_token_accuracy": 0.8337442219257355, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.59963459951537, + "eval_loss": 0.5661377310752869, + "eval_mean_token_accuracy": 0.8304069665500096, + "eval_num_tokens": 977707.0, + "eval_runtime": 104.9926, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 360 + }, + { + "entropy": 0.6145547524094581, + "epoch": 0.9325153374233128, + "grad_norm": 0.3521580696105957, + "learning_rate": 0.00021632824912303506, + "loss": 0.5705258846282959, + "mean_token_accuracy": 0.829042598605156, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.5960987096173422, + "eval_loss": 0.5600873231887817, + "eval_mean_token_accuracy": 0.8332455556733268, + "eval_num_tokens": 1028058.0, + "eval_runtime": 104.8158, + "eval_samples_per_second": 13.309, + "eval_steps_per_second": 1.67, + "step": 380 + }, + { + "entropy": 0.5970950566232205, + "epoch": 0.9815950920245399, + "grad_norm": 0.28969496488571167, + "learning_rate": 0.00020250324669252877, + "loss": 0.5550286293029785, + "mean_token_accuracy": 0.8330720633268356, + "num_tokens": 1083860.0, + "step": 400 + }, + { + "epoch": 0.9815950920245399, + "eval_entropy": 0.590186082805906, + "eval_loss": 0.5563607215881348, + "eval_mean_token_accuracy": 0.833972008228302, + "eval_num_tokens": 1083860.0, + "eval_runtime": 104.9587, + "eval_samples_per_second": 13.291, + "eval_steps_per_second": 1.667, + "step": 400 + }, + { + "entropy": 0.5686816084079254, + "epoch": 1.0294478527607362, + "grad_norm": 0.30075210332870483, + "learning_rate": 0.0001884184670097092, + "loss": 0.5245012760162353, + "mean_token_accuracy": 0.8398806681999793, + "num_tokens": 1136522.0, + "step": 420 + }, + { + "epoch": 1.0294478527607362, + "eval_entropy": 0.5584452276570456, + "eval_loss": 0.557037889957428, + "eval_mean_token_accuracy": 0.8341134299550738, + "eval_num_tokens": 1136522.0, + "eval_runtime": 104.9666, + "eval_samples_per_second": 13.29, + "eval_steps_per_second": 1.667, + "step": 420 + }, + { + "entropy": 0.5519779615104199, + "epoch": 1.0785276073619632, + "grad_norm": 0.27849081158638, + "learning_rate": 0.00017417705609114545, + "loss": 0.5106734275817871, + "mean_token_accuracy": 0.8430664092302322, + "num_tokens": 1196771.0, + "step": 440 + }, + { + "epoch": 1.0785276073619632, + "eval_entropy": 0.5666922775336675, + "eval_loss": 0.5501689910888672, + "eval_mean_token_accuracy": 0.8352916158948626, + "eval_num_tokens": 1196771.0, + "eval_runtime": 104.9138, + "eval_samples_per_second": 13.297, + "eval_steps_per_second": 1.668, + "step": 440 + }, + { + "entropy": 0.5478041648864747, + "epoch": 1.1276073619631901, + "grad_norm": 0.31796717643737793, + "learning_rate": 0.00015988330699925318, + "loss": 0.49846343994140624, + "mean_token_accuracy": 0.8460684299468995, + "num_tokens": 1252972.0, + "step": 460 + }, + { + "epoch": 1.1276073619631901, + "eval_entropy": 0.5484867027827671, + "eval_loss": 0.548787534236908, + "eval_mean_token_accuracy": 0.8369865683146885, + "eval_num_tokens": 1252972.0, + "eval_runtime": 104.9494, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 460 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.543918574586368e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3400933e5cb9591a7e96056b17c3be020cc0a141 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/trainer_state.json @@ -0,0 +1,538 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.1766871165644173, + "eval_steps": 20, + "global_step": 480, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + }, + { + "entropy": 0.6182309970259666, + "epoch": 0.6871165644171779, + "grad_norm": 0.3791189193725586, + "learning_rate": 0.0002781205962525053, + "loss": 0.5778349876403809, + "mean_token_accuracy": 0.8246920928359032, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6430293495314462, + "eval_loss": 0.584140419960022, + "eval_mean_token_accuracy": 0.8250665313856942, + "eval_num_tokens": 760708.0, + "eval_runtime": 105.0023, + "eval_samples_per_second": 13.285, + "eval_steps_per_second": 1.667, + "step": 280 + }, + { + "entropy": 0.6252344503998757, + "epoch": 0.7361963190184049, + "grad_norm": 0.3159662187099457, + "learning_rate": 0.0002670495794561784, + "loss": 0.5845423221588135, + "mean_token_accuracy": 0.8252305090427399, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6507578223092215, + "eval_loss": 0.5797898173332214, + "eval_mean_token_accuracy": 0.8279903977257865, + "eval_num_tokens": 816117.0, + "eval_runtime": 104.8929, + "eval_samples_per_second": 13.299, + "eval_steps_per_second": 1.668, + "step": 300 + }, + { + "entropy": 0.6199106499552727, + "epoch": 0.7852760736196319, + "grad_norm": 0.33928635716438293, + "learning_rate": 0.0002552460980628558, + "loss": 0.5756847858428955, + "mean_token_accuracy": 0.8251789346337318, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6414087894984654, + "eval_loss": 0.5759831070899963, + "eval_mean_token_accuracy": 0.8279267614228385, + "eval_num_tokens": 869286.0, + "eval_runtime": 104.9796, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 320 + }, + { + "entropy": 0.6080872133374214, + "epoch": 0.8343558282208589, + "grad_norm": 0.2935579717159271, + "learning_rate": 0.00024279659162748857, + "loss": 0.56481032371521, + "mean_token_accuracy": 0.8329314738512039, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6243992117473057, + "eval_loss": 0.5680164694786072, + "eval_mean_token_accuracy": 0.8308778762817383, + "eval_num_tokens": 922579.0, + "eval_runtime": 104.9853, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 340 + }, + { + "entropy": 0.6098237752914428, + "epoch": 0.8834355828220859, + "grad_norm": 0.3914716839790344, + "learning_rate": 0.0002297922306920359, + "loss": 0.559494924545288, + "mean_token_accuracy": 0.8337442219257355, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.59963459951537, + "eval_loss": 0.5661377310752869, + "eval_mean_token_accuracy": 0.8304069665500096, + "eval_num_tokens": 977707.0, + "eval_runtime": 104.9926, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 360 + }, + { + "entropy": 0.6145547524094581, + "epoch": 0.9325153374233128, + "grad_norm": 0.3521580696105957, + "learning_rate": 0.00021632824912303506, + "loss": 0.5705258846282959, + "mean_token_accuracy": 0.829042598605156, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.5960987096173422, + "eval_loss": 0.5600873231887817, + "eval_mean_token_accuracy": 0.8332455556733268, + "eval_num_tokens": 1028058.0, + "eval_runtime": 104.8158, + "eval_samples_per_second": 13.309, + "eval_steps_per_second": 1.67, + "step": 380 + }, + { + "entropy": 0.5970950566232205, + "epoch": 0.9815950920245399, + "grad_norm": 0.28969496488571167, + "learning_rate": 0.00020250324669252877, + "loss": 0.5550286293029785, + "mean_token_accuracy": 0.8330720633268356, + "num_tokens": 1083860.0, + "step": 400 + }, + { + "epoch": 0.9815950920245399, + "eval_entropy": 0.590186082805906, + "eval_loss": 0.5563607215881348, + "eval_mean_token_accuracy": 0.833972008228302, + "eval_num_tokens": 1083860.0, + "eval_runtime": 104.9587, + "eval_samples_per_second": 13.291, + "eval_steps_per_second": 1.667, + "step": 400 + }, + { + "entropy": 0.5686816084079254, + "epoch": 1.0294478527607362, + "grad_norm": 0.30075210332870483, + "learning_rate": 0.0001884184670097092, + "loss": 0.5245012760162353, + "mean_token_accuracy": 0.8398806681999793, + "num_tokens": 1136522.0, + "step": 420 + }, + { + "epoch": 1.0294478527607362, + "eval_entropy": 0.5584452276570456, + "eval_loss": 0.557037889957428, + "eval_mean_token_accuracy": 0.8341134299550738, + "eval_num_tokens": 1136522.0, + "eval_runtime": 104.9666, + "eval_samples_per_second": 13.29, + "eval_steps_per_second": 1.667, + "step": 420 + }, + { + "entropy": 0.5519779615104199, + "epoch": 1.0785276073619632, + "grad_norm": 0.27849081158638, + "learning_rate": 0.00017417705609114545, + "loss": 0.5106734275817871, + "mean_token_accuracy": 0.8430664092302322, + "num_tokens": 1196771.0, + "step": 440 + }, + { + "epoch": 1.0785276073619632, + "eval_entropy": 0.5666922775336675, + "eval_loss": 0.5501689910888672, + "eval_mean_token_accuracy": 0.8352916158948626, + "eval_num_tokens": 1196771.0, + "eval_runtime": 104.9138, + "eval_samples_per_second": 13.297, + "eval_steps_per_second": 1.668, + "step": 440 + }, + { + "entropy": 0.5478041648864747, + "epoch": 1.1276073619631901, + "grad_norm": 0.31796717643737793, + "learning_rate": 0.00015988330699925318, + "loss": 0.49846343994140624, + "mean_token_accuracy": 0.8460684299468995, + "num_tokens": 1252972.0, + "step": 460 + }, + { + "epoch": 1.1276073619631901, + "eval_entropy": 0.5484867027827671, + "eval_loss": 0.548787534236908, + "eval_mean_token_accuracy": 0.8369865683146885, + "eval_num_tokens": 1252972.0, + "eval_runtime": 104.9494, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 460 + }, + { + "entropy": 0.5445090994238854, + "epoch": 1.1766871165644173, + "grad_norm": 0.29796338081359863, + "learning_rate": 0.00014564189608068934, + "loss": 0.4977872848510742, + "mean_token_accuracy": 0.8480902388691902, + "num_tokens": 1304559.0, + "step": 480 + }, + { + "epoch": 1.1766871165644173, + "eval_entropy": 0.5576169657707214, + "eval_loss": 0.5454818606376648, + "eval_mean_token_accuracy": 0.8373282974106925, + "eval_num_tokens": 1304559.0, + "eval_runtime": 104.9398, + "eval_samples_per_second": 13.293, + "eval_steps_per_second": 1.668, + "step": 480 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.646427352185344e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0aff30226e312e1c13cad105f8082e4a5aa2aa80 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/trainer_state.json @@ -0,0 +1,559 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.2257668711656442, + "eval_steps": 20, + "global_step": 500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + }, + { + "entropy": 0.729012505710125, + "epoch": 0.24539877300613497, + "grad_norm": 0.6172338724136353, + "learning_rate": 0.0003336184073169935, + "loss": 0.6861891746520996, + "mean_token_accuracy": 0.8014167010784149, + "num_tokens": 267691.0, + "step": 100 + }, + { + "epoch": 0.24539877300613497, + "eval_entropy": 0.7107754983220782, + "eval_loss": 0.6713247299194336, + "eval_mean_token_accuracy": 0.8043575610433306, + "eval_num_tokens": 267691.0, + "eval_runtime": 104.9968, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 100 + }, + { + "entropy": 0.6980620548129082, + "epoch": 0.294478527607362, + "grad_norm": 0.39251449704170227, + "learning_rate": 0.0003319702576276399, + "loss": 0.6606289863586425, + "mean_token_accuracy": 0.8069421723484993, + "num_tokens": 323653.0, + "step": 120 + }, + { + "epoch": 0.294478527607362, + "eval_entropy": 0.6927587161745344, + "eval_loss": 0.6542542576789856, + "eval_mean_token_accuracy": 0.8079089961733137, + "eval_num_tokens": 323653.0, + "eval_runtime": 104.9951, + "eval_samples_per_second": 13.286, + "eval_steps_per_second": 1.667, + "step": 120 + }, + { + "entropy": 0.6662331499159336, + "epoch": 0.34355828220858897, + "grad_norm": 0.3216697573661804, + "learning_rate": 0.0003291142145808027, + "loss": 0.6278485298156739, + "mean_token_accuracy": 0.8151775613427162, + "num_tokens": 382063.0, + "step": 140 + }, + { + "epoch": 0.34355828220858897, + "eval_entropy": 0.6883142059189933, + "eval_loss": 0.6383033990859985, + "eval_mean_token_accuracy": 0.8143898207800729, + "eval_num_tokens": 382063.0, + "eval_runtime": 105.044, + "eval_samples_per_second": 13.28, + "eval_steps_per_second": 1.666, + "step": 140 + }, + { + "entropy": 0.6582960978150367, + "epoch": 0.39263803680981596, + "grad_norm": 0.3135772943496704, + "learning_rate": 0.00032507119362351535, + "loss": 0.6200582027435303, + "mean_token_accuracy": 0.8181118443608284, + "num_tokens": 440825.0, + "step": 160 + }, + { + "epoch": 0.39263803680981596, + "eval_entropy": 0.6579666543006897, + "eval_loss": 0.6228571534156799, + "eval_mean_token_accuracy": 0.8177092627116612, + "eval_num_tokens": 440825.0, + "eval_runtime": 105.0261, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 160 + }, + { + "entropy": 0.6583003848791122, + "epoch": 0.44171779141104295, + "grad_norm": 0.4423973858356476, + "learning_rate": 0.0003198708027096144, + "loss": 0.620824670791626, + "mean_token_accuracy": 0.8163372203707695, + "num_tokens": 494124.0, + "step": 180 + }, + { + "epoch": 0.44171779141104295, + "eval_entropy": 0.6453603114400591, + "eval_loss": 0.6144688129425049, + "eval_mean_token_accuracy": 0.8197193598747253, + "eval_num_tokens": 494124.0, + "eval_runtime": 105.0276, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 180 + }, + { + "entropy": 0.6454010501503944, + "epoch": 0.49079754601226994, + "grad_norm": 0.3798937201499939, + "learning_rate": 0.00031355112547402, + "loss": 0.6067435264587402, + "mean_token_accuracy": 0.8212858110666275, + "num_tokens": 542703.0, + "step": 200 + }, + { + "epoch": 0.49079754601226994, + "eval_entropy": 0.6678816018785749, + "eval_loss": 0.6147017478942871, + "eval_mean_token_accuracy": 0.8175191504614694, + "eval_num_tokens": 542703.0, + "eval_runtime": 105.0272, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 200 + }, + { + "entropy": 0.6504128783941269, + "epoch": 0.5398773006134969, + "grad_norm": 0.4333654046058655, + "learning_rate": 0.00030615844233769247, + "loss": 0.609531021118164, + "mean_token_accuracy": 0.8201618298888207, + "num_tokens": 597461.0, + "step": 220 + }, + { + "epoch": 0.5398773006134969, + "eval_entropy": 0.6659639770644051, + "eval_loss": 0.6074424982070923, + "eval_mean_token_accuracy": 0.8188560080528259, + "eval_num_tokens": 597461.0, + "eval_runtime": 104.9219, + "eval_samples_per_second": 13.296, + "eval_steps_per_second": 1.668, + "step": 220 + }, + { + "entropy": 0.6530551195144654, + "epoch": 0.588957055214724, + "grad_norm": 0.3736813962459564, + "learning_rate": 0.00029774689158567713, + "loss": 0.6050861358642579, + "mean_token_accuracy": 0.819097925722599, + "num_tokens": 651354.0, + "step": 240 + }, + { + "epoch": 0.588957055214724, + "eval_entropy": 0.6647280216217041, + "eval_loss": 0.6030699014663696, + "eval_mean_token_accuracy": 0.8212648391723633, + "eval_num_tokens": 651354.0, + "eval_runtime": 104.9747, + "eval_samples_per_second": 13.289, + "eval_steps_per_second": 1.667, + "step": 240 + }, + { + "entropy": 0.6175086311995983, + "epoch": 0.6380368098159509, + "grad_norm": 0.3385840356349945, + "learning_rate": 0.0002883780729002393, + "loss": 0.5826203346252441, + "mean_token_accuracy": 0.8260212540626526, + "num_tokens": 707219.0, + "step": 260 + }, + { + "epoch": 0.6380368098159509, + "eval_entropy": 0.6298109524590628, + "eval_loss": 0.5883784890174866, + "eval_mean_token_accuracy": 0.8267830569403513, + "eval_num_tokens": 707219.0, + "eval_runtime": 104.9491, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 260 + }, + { + "entropy": 0.6182309970259666, + "epoch": 0.6871165644171779, + "grad_norm": 0.3791189193725586, + "learning_rate": 0.0002781205962525053, + "loss": 0.5778349876403809, + "mean_token_accuracy": 0.8246920928359032, + "num_tokens": 760708.0, + "step": 280 + }, + { + "epoch": 0.6871165644171779, + "eval_entropy": 0.6430293495314462, + "eval_loss": 0.584140419960022, + "eval_mean_token_accuracy": 0.8250665313856942, + "eval_num_tokens": 760708.0, + "eval_runtime": 105.0023, + "eval_samples_per_second": 13.285, + "eval_steps_per_second": 1.667, + "step": 280 + }, + { + "entropy": 0.6252344503998757, + "epoch": 0.7361963190184049, + "grad_norm": 0.3159662187099457, + "learning_rate": 0.0002670495794561784, + "loss": 0.5845423221588135, + "mean_token_accuracy": 0.8252305090427399, + "num_tokens": 816117.0, + "step": 300 + }, + { + "epoch": 0.7361963190184049, + "eval_entropy": 0.6507578223092215, + "eval_loss": 0.5797898173332214, + "eval_mean_token_accuracy": 0.8279903977257865, + "eval_num_tokens": 816117.0, + "eval_runtime": 104.8929, + "eval_samples_per_second": 13.299, + "eval_steps_per_second": 1.668, + "step": 300 + }, + { + "entropy": 0.6199106499552727, + "epoch": 0.7852760736196319, + "grad_norm": 0.33928635716438293, + "learning_rate": 0.0002552460980628558, + "loss": 0.5756847858428955, + "mean_token_accuracy": 0.8251789346337318, + "num_tokens": 869286.0, + "step": 320 + }, + { + "epoch": 0.7852760736196319, + "eval_entropy": 0.6414087894984654, + "eval_loss": 0.5759831070899963, + "eval_mean_token_accuracy": 0.8279267614228385, + "eval_num_tokens": 869286.0, + "eval_runtime": 104.9796, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 320 + }, + { + "entropy": 0.6080872133374214, + "epoch": 0.8343558282208589, + "grad_norm": 0.2935579717159271, + "learning_rate": 0.00024279659162748857, + "loss": 0.56481032371521, + "mean_token_accuracy": 0.8329314738512039, + "num_tokens": 922579.0, + "step": 340 + }, + { + "epoch": 0.8343558282208589, + "eval_entropy": 0.6243992117473057, + "eval_loss": 0.5680164694786072, + "eval_mean_token_accuracy": 0.8308778762817383, + "eval_num_tokens": 922579.0, + "eval_runtime": 104.9853, + "eval_samples_per_second": 13.288, + "eval_steps_per_second": 1.667, + "step": 340 + }, + { + "entropy": 0.6098237752914428, + "epoch": 0.8834355828220859, + "grad_norm": 0.3914716839790344, + "learning_rate": 0.0002297922306920359, + "loss": 0.559494924545288, + "mean_token_accuracy": 0.8337442219257355, + "num_tokens": 977707.0, + "step": 360 + }, + { + "epoch": 0.8834355828220859, + "eval_entropy": 0.59963459951537, + "eval_loss": 0.5661377310752869, + "eval_mean_token_accuracy": 0.8304069665500096, + "eval_num_tokens": 977707.0, + "eval_runtime": 104.9926, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 360 + }, + { + "entropy": 0.6145547524094581, + "epoch": 0.9325153374233128, + "grad_norm": 0.3521580696105957, + "learning_rate": 0.00021632824912303506, + "loss": 0.5705258846282959, + "mean_token_accuracy": 0.829042598605156, + "num_tokens": 1028058.0, + "step": 380 + }, + { + "epoch": 0.9325153374233128, + "eval_entropy": 0.5960987096173422, + "eval_loss": 0.5600873231887817, + "eval_mean_token_accuracy": 0.8332455556733268, + "eval_num_tokens": 1028058.0, + "eval_runtime": 104.8158, + "eval_samples_per_second": 13.309, + "eval_steps_per_second": 1.67, + "step": 380 + }, + { + "entropy": 0.5970950566232205, + "epoch": 0.9815950920245399, + "grad_norm": 0.28969496488571167, + "learning_rate": 0.00020250324669252877, + "loss": 0.5550286293029785, + "mean_token_accuracy": 0.8330720633268356, + "num_tokens": 1083860.0, + "step": 400 + }, + { + "epoch": 0.9815950920245399, + "eval_entropy": 0.590186082805906, + "eval_loss": 0.5563607215881348, + "eval_mean_token_accuracy": 0.833972008228302, + "eval_num_tokens": 1083860.0, + "eval_runtime": 104.9587, + "eval_samples_per_second": 13.291, + "eval_steps_per_second": 1.667, + "step": 400 + }, + { + "entropy": 0.5686816084079254, + "epoch": 1.0294478527607362, + "grad_norm": 0.30075210332870483, + "learning_rate": 0.0001884184670097092, + "loss": 0.5245012760162353, + "mean_token_accuracy": 0.8398806681999793, + "num_tokens": 1136522.0, + "step": 420 + }, + { + "epoch": 1.0294478527607362, + "eval_entropy": 0.5584452276570456, + "eval_loss": 0.557037889957428, + "eval_mean_token_accuracy": 0.8341134299550738, + "eval_num_tokens": 1136522.0, + "eval_runtime": 104.9666, + "eval_samples_per_second": 13.29, + "eval_steps_per_second": 1.667, + "step": 420 + }, + { + "entropy": 0.5519779615104199, + "epoch": 1.0785276073619632, + "grad_norm": 0.27849081158638, + "learning_rate": 0.00017417705609114545, + "loss": 0.5106734275817871, + "mean_token_accuracy": 0.8430664092302322, + "num_tokens": 1196771.0, + "step": 440 + }, + { + "epoch": 1.0785276073619632, + "eval_entropy": 0.5666922775336675, + "eval_loss": 0.5501689910888672, + "eval_mean_token_accuracy": 0.8352916158948626, + "eval_num_tokens": 1196771.0, + "eval_runtime": 104.9138, + "eval_samples_per_second": 13.297, + "eval_steps_per_second": 1.668, + "step": 440 + }, + { + "entropy": 0.5478041648864747, + "epoch": 1.1276073619631901, + "grad_norm": 0.31796717643737793, + "learning_rate": 0.00015988330699925318, + "loss": 0.49846343994140624, + "mean_token_accuracy": 0.8460684299468995, + "num_tokens": 1252972.0, + "step": 460 + }, + { + "epoch": 1.1276073619631901, + "eval_entropy": 0.5484867027827671, + "eval_loss": 0.548787534236908, + "eval_mean_token_accuracy": 0.8369865683146885, + "eval_num_tokens": 1252972.0, + "eval_runtime": 104.9494, + "eval_samples_per_second": 13.292, + "eval_steps_per_second": 1.667, + "step": 460 + }, + { + "entropy": 0.5445090994238854, + "epoch": 1.1766871165644173, + "grad_norm": 0.29796338081359863, + "learning_rate": 0.00014564189608068934, + "loss": 0.4977872848510742, + "mean_token_accuracy": 0.8480902388691902, + "num_tokens": 1304559.0, + "step": 480 + }, + { + "epoch": 1.1766871165644173, + "eval_entropy": 0.5576169657707214, + "eval_loss": 0.5454818606376648, + "eval_mean_token_accuracy": 0.8373282974106925, + "eval_num_tokens": 1304559.0, + "eval_runtime": 104.9398, + "eval_samples_per_second": 13.293, + "eval_steps_per_second": 1.668, + "step": 480 + }, + { + "entropy": 0.5288209930062294, + "epoch": 1.2257668711656442, + "grad_norm": 0.3287126123905182, + "learning_rate": 0.00013155711639786983, + "loss": 0.4876402854919434, + "mean_token_accuracy": 0.8504441276192665, + "num_tokens": 1362151.0, + "step": 500 + }, + { + "epoch": 1.2257668711656442, + "eval_entropy": 0.5497624914986746, + "eval_loss": 0.5430853366851807, + "eval_mean_token_accuracy": 0.837989068712507, + "eval_num_tokens": 1362151.0, + "eval_runtime": 104.9569, + "eval_samples_per_second": 13.291, + "eval_steps_per_second": 1.667, + "step": 500 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.7562422835282944e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1905fe136679d2f84930a8fff082089f77b4a7ad --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/trainer_state.json @@ -0,0 +1,97 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.147239263803681, + "eval_steps": 20, + "global_step": 60, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.450420492976128e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/README.md b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/README.md new file mode 100644 index 0000000000000000000000000000000000000000..836e06c597a999d7e5c695115d88badf693532cc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-14B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7714cc1f043abaa59754d1f3150db1bde45c00dc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.09184864657147984, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a9bc4e672b995b0f4c89736a153db4707fdf5d52 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3-14B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/trainer_state.json @@ -0,0 +1,118 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.19631901840490798, + "eval_steps": 20, + "global_step": 80, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.719682201743126, + "epoch": 0.049079754601226995, + "grad_norm": 0.8067639470100403, + "learning_rate": 7.740423047216553e-05, + "loss": 1.700193214416504, + "mean_token_accuracy": 0.6428261503577233, + "num_tokens": 54280.0, + "step": 20 + }, + { + "epoch": 0.049079754601226995, + "eval_entropy": 1.315366074017116, + "eval_loss": 1.23488187789917, + "eval_mean_token_accuracy": 0.7063105702400208, + "eval_num_tokens": 54280.0, + "eval_runtime": 105.4535, + "eval_samples_per_second": 13.229, + "eval_steps_per_second": 1.659, + "step": 20 + }, + { + "entropy": 0.95277059674263, + "epoch": 0.09815950920245399, + "grad_norm": 0.572104811668396, + "learning_rate": 0.00015888236781128713, + "loss": 0.9082255363464355, + "mean_token_accuracy": 0.7559243977069855, + "num_tokens": 113362.0, + "step": 40 + }, + { + "epoch": 0.09815950920245399, + "eval_entropy": 0.8220351917403085, + "eval_loss": 0.7917433381080627, + "eval_mean_token_accuracy": 0.774279066153935, + "eval_num_tokens": 113362.0, + "eval_runtime": 104.9915, + "eval_samples_per_second": 13.287, + "eval_steps_per_second": 1.667, + "step": 40 + }, + { + "entropy": 0.7833507835865021, + "epoch": 0.147239263803681, + "grad_norm": 0.5184682011604309, + "learning_rate": 0.00024036050515040874, + "loss": 0.7395487308502198, + "mean_token_accuracy": 0.7903637677431107, + "num_tokens": 165819.0, + "step": 60 + }, + { + "epoch": 0.147239263803681, + "eval_entropy": 0.7486384316853114, + "eval_loss": 0.7170758843421936, + "eval_mean_token_accuracy": 0.7953836243493216, + "eval_num_tokens": 165819.0, + "eval_runtime": 105.0262, + "eval_samples_per_second": 13.282, + "eval_steps_per_second": 1.666, + "step": 60 + }, + { + "entropy": 0.7294519171118736, + "epoch": 0.19631901840490798, + "grad_norm": 0.44408509135246277, + "learning_rate": 0.00032183864248953035, + "loss": 0.6858654499053956, + "mean_token_accuracy": 0.8006252631545067, + "num_tokens": 215870.0, + "step": 80 + }, + { + "epoch": 0.19631901840490798, + "eval_entropy": 0.7126145311764308, + "eval_loss": 0.6875877976417542, + "eval_mean_token_accuracy": 0.8029442460196359, + "eval_num_tokens": 215870.0, + "eval_runtime": 105.0224, + "eval_samples_per_second": 13.283, + "eval_steps_per_second": 1.666, + "step": 80 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.579400021188608e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}