diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6d49c963566763e019105397376e021d04979f13 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: transformers +model_name: Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1 + +This model is a fine-tuned version of [Qwen/Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/j4uxftxv) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d816bf77c304d3fdca665ab80153fe1c56d5d502 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: transformers +model_name: Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2 + +This model is a fine-tuned version of [Qwen/Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/lqx6ngjd) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..aa93a30589618aa8d3ff6510bfdf5a0fdf35a200 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/trainer_state.json @@ -0,0 +1,139 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.24554941682013506, + "eval_steps": 20, + "global_step": 100, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4188582379977216.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..12b7153d6508f0e58a71ae60b595012a8974c052 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-120/trainer_state.json @@ -0,0 +1,160 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2946593001841621, + "eval_steps": 20, + "global_step": 120, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5018169392093184.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3b4c5e3d4362c582da22a05cefdd0f944c94d1e3 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-140/trainer_state.json @@ -0,0 +1,181 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.3437691835481891, + "eval_steps": 20, + "global_step": 140, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5844207297243648.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3afff98e913a446118700662cccab11f49546779 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-160/trainer_state.json @@ -0,0 +1,202 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.3928790669122161, + "eval_steps": 20, + "global_step": 160, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6699341183083008.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8f88849a35e14c410f5661ce591514c35ac329bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-180/trainer_state.json @@ -0,0 +1,223 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.4419889502762431, + "eval_steps": 20, + "global_step": 180, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7464542037702144.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..af31efd65e3fd465a5333ccf64c02d974de045a7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-20/trainer_state.json @@ -0,0 +1,55 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.04910988336402701, + "eval_steps": 20, + "global_step": 20, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 842611565036544.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8744dff90b6de08b111daa0d35d5bb72a1558aa8 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-200/trainer_state.json @@ -0,0 +1,244 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.4910988336402701, + "eval_steps": 20, + "global_step": 200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8217555392930304.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..19bfad854bc7e2e9757214158697f5c639eb4e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-220/trainer_state.json @@ -0,0 +1,265 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5402087170042971, + "eval_steps": 20, + "global_step": 220, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9058794190178304.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a9eb77a2bd8a257654d2f5f19d84d0621ebcea32 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-240/trainer_state.json @@ -0,0 +1,286 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5893186003683242, + "eval_steps": 20, + "global_step": 240, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9845055311602176.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2ba0e2334377ebc03019cbc4a08be2f09885328d --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-260/trainer_state.json @@ -0,0 +1,307 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6384284837323512, + "eval_steps": 20, + "global_step": 260, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.072031196429312e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1d197ad5f0b06fda8e392f0f2aa34e9528228781 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-280/trainer_state.json @@ -0,0 +1,328 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6875383670963782, + "eval_steps": 20, + "global_step": 280, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + }, + { + "entropy": 0.781412948668003, + "epoch": 0.6875383670963782, + "grad_norm": 1.1252366304397583, + "learning_rate": 6.980686174774777e-05, + "loss": 0.6978650093078613, + "mean_token_accuracy": 0.7952493160963059, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7790967045511519, + "eval_loss": 0.7110744118690491, + "eval_mean_token_accuracy": 0.7964006056104388, + "eval_num_tokens": 720146.0, + "eval_runtime": 53.9404, + "eval_samples_per_second": 25.862, + "eval_steps_per_second": 3.244, + "step": 280 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.1502052996657152e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4f8f0d5fada321e92cd62ebfbf11b105a328069f --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-300/trainer_state.json @@ -0,0 +1,349 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7366482504604052, + "eval_steps": 20, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + }, + { + "entropy": 0.781412948668003, + "epoch": 0.6875383670963782, + "grad_norm": 1.1252366304397583, + "learning_rate": 6.980686174774777e-05, + "loss": 0.6978650093078613, + "mean_token_accuracy": 0.7952493160963059, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7790967045511519, + "eval_loss": 0.7110744118690491, + "eval_mean_token_accuracy": 0.7964006056104388, + "eval_num_tokens": 720146.0, + "eval_runtime": 53.9404, + "eval_samples_per_second": 25.862, + "eval_steps_per_second": 3.244, + "step": 280 + }, + { + "entropy": 0.7788082852959632, + "epoch": 0.7366482504604052, + "grad_norm": 1.1519086360931396, + "learning_rate": 6.702809257595105e-05, + "loss": 0.7019347190856934, + "mean_token_accuracy": 0.7960198536515236, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7932151372092111, + "eval_loss": 0.7035816311836243, + "eval_mean_token_accuracy": 0.7979611914498466, + "eval_num_tokens": 772624.0, + "eval_runtime": 54.0517, + "eval_samples_per_second": 25.809, + "eval_steps_per_second": 3.238, + "step": 300 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.236294581078016e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9c4d7be332c7b92f01be16755bcc74201adb8594 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-320/trainer_state.json @@ -0,0 +1,370 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7857581338244322, + "eval_steps": 20, + "global_step": 320, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + }, + { + "entropy": 0.781412948668003, + "epoch": 0.6875383670963782, + "grad_norm": 1.1252366304397583, + "learning_rate": 6.980686174774777e-05, + "loss": 0.6978650093078613, + "mean_token_accuracy": 0.7952493160963059, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7790967045511519, + "eval_loss": 0.7110744118690491, + "eval_mean_token_accuracy": 0.7964006056104388, + "eval_num_tokens": 720146.0, + "eval_runtime": 53.9404, + "eval_samples_per_second": 25.862, + "eval_steps_per_second": 3.244, + "step": 280 + }, + { + "entropy": 0.7788082852959632, + "epoch": 0.7366482504604052, + "grad_norm": 1.1519086360931396, + "learning_rate": 6.702809257595105e-05, + "loss": 0.7019347190856934, + "mean_token_accuracy": 0.7960198536515236, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7932151372092111, + "eval_loss": 0.7035816311836243, + "eval_mean_token_accuracy": 0.7979611914498466, + "eval_num_tokens": 772624.0, + "eval_runtime": 54.0517, + "eval_samples_per_second": 25.809, + "eval_steps_per_second": 3.238, + "step": 300 + }, + { + "entropy": 0.7729029789566993, + "epoch": 0.7857581338244322, + "grad_norm": 1.0247046947479248, + "learning_rate": 6.406547849821584e-05, + "loss": 0.6924194812774658, + "mean_token_accuracy": 0.7985413707792759, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7812128639221192, + "eval_loss": 0.6989337205886841, + "eval_mean_token_accuracy": 0.8005841762678964, + "eval_num_tokens": 822969.0, + "eval_runtime": 54.1306, + "eval_samples_per_second": 25.771, + "eval_steps_per_second": 3.233, + "step": 320 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.3174452466656768e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0d1728526342aef6c0b44fad8e2e7b7c2a14f04f --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-340/trainer_state.json @@ -0,0 +1,391 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8348680171884592, + "eval_steps": 20, + "global_step": 340, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + }, + { + "entropy": 0.781412948668003, + "epoch": 0.6875383670963782, + "grad_norm": 1.1252366304397583, + "learning_rate": 6.980686174774777e-05, + "loss": 0.6978650093078613, + "mean_token_accuracy": 0.7952493160963059, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7790967045511519, + "eval_loss": 0.7110744118690491, + "eval_mean_token_accuracy": 0.7964006056104388, + "eval_num_tokens": 720146.0, + "eval_runtime": 53.9404, + "eval_samples_per_second": 25.862, + "eval_steps_per_second": 3.244, + "step": 280 + }, + { + "entropy": 0.7788082852959632, + "epoch": 0.7366482504604052, + "grad_norm": 1.1519086360931396, + "learning_rate": 6.702809257595105e-05, + "loss": 0.7019347190856934, + "mean_token_accuracy": 0.7960198536515236, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7932151372092111, + "eval_loss": 0.7035816311836243, + "eval_mean_token_accuracy": 0.7979611914498466, + "eval_num_tokens": 772624.0, + "eval_runtime": 54.0517, + "eval_samples_per_second": 25.809, + "eval_steps_per_second": 3.238, + "step": 300 + }, + { + "entropy": 0.7729029789566993, + "epoch": 0.7857581338244322, + "grad_norm": 1.0247046947479248, + "learning_rate": 6.406547849821584e-05, + "loss": 0.6924194812774658, + "mean_token_accuracy": 0.7985413707792759, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7812128639221192, + "eval_loss": 0.6989337205886841, + "eval_mean_token_accuracy": 0.8005841762678964, + "eval_num_tokens": 822969.0, + "eval_runtime": 54.1306, + "eval_samples_per_second": 25.771, + "eval_steps_per_second": 3.233, + "step": 320 + }, + { + "entropy": 0.7627703189849854, + "epoch": 0.8348680171884592, + "grad_norm": 0.957506537437439, + "learning_rate": 6.0940715405257566e-05, + "loss": 0.679268217086792, + "mean_token_accuracy": 0.8040587857365609, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7624005011149815, + "eval_loss": 0.6934819221496582, + "eval_mean_token_accuracy": 0.8014607231957572, + "eval_num_tokens": 873459.0, + "eval_runtime": 54.1656, + "eval_samples_per_second": 25.754, + "eval_steps_per_second": 3.231, + "step": 340 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.4004474746608128e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3199f4b65917223a1af32f00dbadf5fd00730623 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-360/trainer_state.json @@ -0,0 +1,412 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8839779005524862, + "eval_steps": 20, + "global_step": 360, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + }, + { + "entropy": 0.781412948668003, + "epoch": 0.6875383670963782, + "grad_norm": 1.1252366304397583, + "learning_rate": 6.980686174774777e-05, + "loss": 0.6978650093078613, + "mean_token_accuracy": 0.7952493160963059, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7790967045511519, + "eval_loss": 0.7110744118690491, + "eval_mean_token_accuracy": 0.7964006056104388, + "eval_num_tokens": 720146.0, + "eval_runtime": 53.9404, + "eval_samples_per_second": 25.862, + "eval_steps_per_second": 3.244, + "step": 280 + }, + { + "entropy": 0.7788082852959632, + "epoch": 0.7366482504604052, + "grad_norm": 1.1519086360931396, + "learning_rate": 6.702809257595105e-05, + "loss": 0.7019347190856934, + "mean_token_accuracy": 0.7960198536515236, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7932151372092111, + "eval_loss": 0.7035816311836243, + "eval_mean_token_accuracy": 0.7979611914498466, + "eval_num_tokens": 772624.0, + "eval_runtime": 54.0517, + "eval_samples_per_second": 25.809, + "eval_steps_per_second": 3.238, + "step": 300 + }, + { + "entropy": 0.7729029789566993, + "epoch": 0.7857581338244322, + "grad_norm": 1.0247046947479248, + "learning_rate": 6.406547849821584e-05, + "loss": 0.6924194812774658, + "mean_token_accuracy": 0.7985413707792759, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7812128639221192, + "eval_loss": 0.6989337205886841, + "eval_mean_token_accuracy": 0.8005841762678964, + "eval_num_tokens": 822969.0, + "eval_runtime": 54.1306, + "eval_samples_per_second": 25.771, + "eval_steps_per_second": 3.233, + "step": 320 + }, + { + "entropy": 0.7627703189849854, + "epoch": 0.8348680171884592, + "grad_norm": 0.957506537437439, + "learning_rate": 6.0940715405257566e-05, + "loss": 0.679268217086792, + "mean_token_accuracy": 0.8040587857365609, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7624005011149815, + "eval_loss": 0.6934819221496582, + "eval_mean_token_accuracy": 0.8014607231957572, + "eval_num_tokens": 873459.0, + "eval_runtime": 54.1656, + "eval_samples_per_second": 25.754, + "eval_steps_per_second": 3.231, + "step": 340 + }, + { + "entropy": 0.763699171692133, + "epoch": 0.8839779005524862, + "grad_norm": 1.4329766035079956, + "learning_rate": 5.767668664158135e-05, + "loss": 0.6764235019683837, + "mean_token_accuracy": 0.8051542706787587, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.7163350166593279, + "eval_loss": 0.6882685422897339, + "eval_mean_token_accuracy": 0.802961312021528, + "eval_num_tokens": 925669.0, + "eval_runtime": 53.8288, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 360 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.4841997026184704e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..000d57a23f3f62b61b291dd6132fa5af6feade8c --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-380/trainer_state.json @@ -0,0 +1,433 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9330877839165131, + "eval_steps": 20, + "global_step": 380, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + }, + { + "entropy": 0.781412948668003, + "epoch": 0.6875383670963782, + "grad_norm": 1.1252366304397583, + "learning_rate": 6.980686174774777e-05, + "loss": 0.6978650093078613, + "mean_token_accuracy": 0.7952493160963059, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7790967045511519, + "eval_loss": 0.7110744118690491, + "eval_mean_token_accuracy": 0.7964006056104388, + "eval_num_tokens": 720146.0, + "eval_runtime": 53.9404, + "eval_samples_per_second": 25.862, + "eval_steps_per_second": 3.244, + "step": 280 + }, + { + "entropy": 0.7788082852959632, + "epoch": 0.7366482504604052, + "grad_norm": 1.1519086360931396, + "learning_rate": 6.702809257595105e-05, + "loss": 0.7019347190856934, + "mean_token_accuracy": 0.7960198536515236, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7932151372092111, + "eval_loss": 0.7035816311836243, + "eval_mean_token_accuracy": 0.7979611914498466, + "eval_num_tokens": 772624.0, + "eval_runtime": 54.0517, + "eval_samples_per_second": 25.809, + "eval_steps_per_second": 3.238, + "step": 300 + }, + { + "entropy": 0.7729029789566993, + "epoch": 0.7857581338244322, + "grad_norm": 1.0247046947479248, + "learning_rate": 6.406547849821584e-05, + "loss": 0.6924194812774658, + "mean_token_accuracy": 0.7985413707792759, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7812128639221192, + "eval_loss": 0.6989337205886841, + "eval_mean_token_accuracy": 0.8005841762678964, + "eval_num_tokens": 822969.0, + "eval_runtime": 54.1306, + "eval_samples_per_second": 25.771, + "eval_steps_per_second": 3.233, + "step": 320 + }, + { + "entropy": 0.7627703189849854, + "epoch": 0.8348680171884592, + "grad_norm": 0.957506537437439, + "learning_rate": 6.0940715405257566e-05, + "loss": 0.679268217086792, + "mean_token_accuracy": 0.8040587857365609, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7624005011149815, + "eval_loss": 0.6934819221496582, + "eval_mean_token_accuracy": 0.8014607231957572, + "eval_num_tokens": 873459.0, + "eval_runtime": 54.1656, + "eval_samples_per_second": 25.754, + "eval_steps_per_second": 3.231, + "step": 340 + }, + { + "entropy": 0.763699171692133, + "epoch": 0.8839779005524862, + "grad_norm": 1.4329766035079956, + "learning_rate": 5.767668664158135e-05, + "loss": 0.6764235019683837, + "mean_token_accuracy": 0.8051542706787587, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.7163350166593279, + "eval_loss": 0.6882685422897339, + "eval_mean_token_accuracy": 0.802961312021528, + "eval_num_tokens": 925669.0, + "eval_runtime": 53.8288, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 360 + }, + { + "entropy": 0.7710359543561935, + "epoch": 0.9330877839165131, + "grad_norm": 1.141253113746643, + "learning_rate": 5.429729542559189e-05, + "loss": 0.6897628784179688, + "mean_token_accuracy": 0.8005676351487636, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7462973778588431, + "eval_loss": 0.6829195022583008, + "eval_mean_token_accuracy": 0.8036447630609784, + "eval_num_tokens": 973436.0, + "eval_runtime": 53.8276, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 380 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.5620657701559808e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e655b77ab87d35c5fdb1c4a3d31bba25930824c5 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-40/trainer_state.json @@ -0,0 +1,76 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.09821976672805402, + "eval_steps": 20, + "global_step": 40, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1715390539279872.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6bd782449580b5cbed1ba77939143950996f5bc9 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-400/trainer_state.json @@ -0,0 +1,454 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9821976672805403, + "eval_steps": 20, + "global_step": 400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + }, + { + "entropy": 0.781412948668003, + "epoch": 0.6875383670963782, + "grad_norm": 1.1252366304397583, + "learning_rate": 6.980686174774777e-05, + "loss": 0.6978650093078613, + "mean_token_accuracy": 0.7952493160963059, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7790967045511519, + "eval_loss": 0.7110744118690491, + "eval_mean_token_accuracy": 0.7964006056104388, + "eval_num_tokens": 720146.0, + "eval_runtime": 53.9404, + "eval_samples_per_second": 25.862, + "eval_steps_per_second": 3.244, + "step": 280 + }, + { + "entropy": 0.7788082852959632, + "epoch": 0.7366482504604052, + "grad_norm": 1.1519086360931396, + "learning_rate": 6.702809257595105e-05, + "loss": 0.7019347190856934, + "mean_token_accuracy": 0.7960198536515236, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7932151372092111, + "eval_loss": 0.7035816311836243, + "eval_mean_token_accuracy": 0.7979611914498466, + "eval_num_tokens": 772624.0, + "eval_runtime": 54.0517, + "eval_samples_per_second": 25.809, + "eval_steps_per_second": 3.238, + "step": 300 + }, + { + "entropy": 0.7729029789566993, + "epoch": 0.7857581338244322, + "grad_norm": 1.0247046947479248, + "learning_rate": 6.406547849821584e-05, + "loss": 0.6924194812774658, + "mean_token_accuracy": 0.7985413707792759, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7812128639221192, + "eval_loss": 0.6989337205886841, + "eval_mean_token_accuracy": 0.8005841762678964, + "eval_num_tokens": 822969.0, + "eval_runtime": 54.1306, + "eval_samples_per_second": 25.771, + "eval_steps_per_second": 3.233, + "step": 320 + }, + { + "entropy": 0.7627703189849854, + "epoch": 0.8348680171884592, + "grad_norm": 0.957506537437439, + "learning_rate": 6.0940715405257566e-05, + "loss": 0.679268217086792, + "mean_token_accuracy": 0.8040587857365609, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7624005011149815, + "eval_loss": 0.6934819221496582, + "eval_mean_token_accuracy": 0.8014607231957572, + "eval_num_tokens": 873459.0, + "eval_runtime": 54.1656, + "eval_samples_per_second": 25.754, + "eval_steps_per_second": 3.231, + "step": 340 + }, + { + "entropy": 0.763699171692133, + "epoch": 0.8839779005524862, + "grad_norm": 1.4329766035079956, + "learning_rate": 5.767668664158135e-05, + "loss": 0.6764235019683837, + "mean_token_accuracy": 0.8051542706787587, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.7163350166593279, + "eval_loss": 0.6882685422897339, + "eval_mean_token_accuracy": 0.802961312021528, + "eval_num_tokens": 925669.0, + "eval_runtime": 53.8288, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 360 + }, + { + "entropy": 0.7710359543561935, + "epoch": 0.9330877839165131, + "grad_norm": 1.141253113746643, + "learning_rate": 5.429729542559189e-05, + "loss": 0.6897628784179688, + "mean_token_accuracy": 0.8005676351487636, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7462973778588431, + "eval_loss": 0.6829195022583008, + "eval_mean_token_accuracy": 0.8036447630609784, + "eval_num_tokens": 973436.0, + "eval_runtime": 53.8276, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 380 + }, + { + "entropy": 0.7500453963875771, + "epoch": 0.9821976672805403, + "grad_norm": 0.9055600166320801, + "learning_rate": 5.0827289800936774e-05, + "loss": 0.6726306915283203, + "mean_token_accuracy": 0.8026990614831447, + "num_tokens": 1026370.0, + "step": 400 + }, + { + "epoch": 0.9821976672805403, + "eval_entropy": 0.745090776511601, + "eval_loss": 0.6795706152915955, + "eval_mean_token_accuracy": 0.8038556255613054, + "eval_num_tokens": 1026370.0, + "eval_runtime": 53.9485, + "eval_samples_per_second": 25.858, + "eval_steps_per_second": 3.244, + "step": 400 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6467287123538432e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..66519828b443b875b5830e51a361df684030ef4b --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-420/trainer_state.json @@ -0,0 +1,475 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0294659300184161, + "eval_steps": 20, + "global_step": 420, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + }, + { + "entropy": 0.781412948668003, + "epoch": 0.6875383670963782, + "grad_norm": 1.1252366304397583, + "learning_rate": 6.980686174774777e-05, + "loss": 0.6978650093078613, + "mean_token_accuracy": 0.7952493160963059, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7790967045511519, + "eval_loss": 0.7110744118690491, + "eval_mean_token_accuracy": 0.7964006056104388, + "eval_num_tokens": 720146.0, + "eval_runtime": 53.9404, + "eval_samples_per_second": 25.862, + "eval_steps_per_second": 3.244, + "step": 280 + }, + { + "entropy": 0.7788082852959632, + "epoch": 0.7366482504604052, + "grad_norm": 1.1519086360931396, + "learning_rate": 6.702809257595105e-05, + "loss": 0.7019347190856934, + "mean_token_accuracy": 0.7960198536515236, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7932151372092111, + "eval_loss": 0.7035816311836243, + "eval_mean_token_accuracy": 0.7979611914498466, + "eval_num_tokens": 772624.0, + "eval_runtime": 54.0517, + "eval_samples_per_second": 25.809, + "eval_steps_per_second": 3.238, + "step": 300 + }, + { + "entropy": 0.7729029789566993, + "epoch": 0.7857581338244322, + "grad_norm": 1.0247046947479248, + "learning_rate": 6.406547849821584e-05, + "loss": 0.6924194812774658, + "mean_token_accuracy": 0.7985413707792759, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7812128639221192, + "eval_loss": 0.6989337205886841, + "eval_mean_token_accuracy": 0.8005841762678964, + "eval_num_tokens": 822969.0, + "eval_runtime": 54.1306, + "eval_samples_per_second": 25.771, + "eval_steps_per_second": 3.233, + "step": 320 + }, + { + "entropy": 0.7627703189849854, + "epoch": 0.8348680171884592, + "grad_norm": 0.957506537437439, + "learning_rate": 6.0940715405257566e-05, + "loss": 0.679268217086792, + "mean_token_accuracy": 0.8040587857365609, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7624005011149815, + "eval_loss": 0.6934819221496582, + "eval_mean_token_accuracy": 0.8014607231957572, + "eval_num_tokens": 873459.0, + "eval_runtime": 54.1656, + "eval_samples_per_second": 25.754, + "eval_steps_per_second": 3.231, + "step": 340 + }, + { + "entropy": 0.763699171692133, + "epoch": 0.8839779005524862, + "grad_norm": 1.4329766035079956, + "learning_rate": 5.767668664158135e-05, + "loss": 0.6764235019683837, + "mean_token_accuracy": 0.8051542706787587, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.7163350166593279, + "eval_loss": 0.6882685422897339, + "eval_mean_token_accuracy": 0.802961312021528, + "eval_num_tokens": 925669.0, + "eval_runtime": 53.8288, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 360 + }, + { + "entropy": 0.7710359543561935, + "epoch": 0.9330877839165131, + "grad_norm": 1.141253113746643, + "learning_rate": 5.429729542559189e-05, + "loss": 0.6897628784179688, + "mean_token_accuracy": 0.8005676351487636, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7462973778588431, + "eval_loss": 0.6829195022583008, + "eval_mean_token_accuracy": 0.8036447630609784, + "eval_num_tokens": 973436.0, + "eval_runtime": 53.8276, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 380 + }, + { + "entropy": 0.7500453963875771, + "epoch": 0.9821976672805403, + "grad_norm": 0.9055600166320801, + "learning_rate": 5.0827289800936774e-05, + "loss": 0.6726306915283203, + "mean_token_accuracy": 0.8026990614831447, + "num_tokens": 1026370.0, + "step": 400 + }, + { + "epoch": 0.9821976672805403, + "eval_entropy": 0.745090776511601, + "eval_loss": 0.6795706152915955, + "eval_mean_token_accuracy": 0.8038556255613054, + "eval_num_tokens": 1026370.0, + "eval_runtime": 53.9485, + "eval_samples_per_second": 25.858, + "eval_steps_per_second": 3.244, + "step": 400 + }, + { + "entropy": 0.7288912821125675, + "epoch": 1.0294659300184161, + "grad_norm": 0.9075890779495239, + "learning_rate": 4.729208140100435e-05, + "loss": 0.6533905506134033, + "mean_token_accuracy": 0.807745551908171, + "num_tokens": 1076273.0, + "step": 420 + }, + { + "epoch": 1.0294659300184161, + "eval_entropy": 0.7279435137340001, + "eval_loss": 0.6758045554161072, + "eval_mean_token_accuracy": 0.8043134975433349, + "eval_num_tokens": 1076273.0, + "eval_runtime": 53.8173, + "eval_samples_per_second": 25.921, + "eval_steps_per_second": 3.252, + "step": 420 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.7256904829616e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..96127595e01dffabb7d71a9f9054561ab7e38fcc --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-440/trainer_state.json @@ -0,0 +1,496 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0785758133824432, + "eval_steps": 20, + "global_step": 440, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + }, + { + "entropy": 0.781412948668003, + "epoch": 0.6875383670963782, + "grad_norm": 1.1252366304397583, + "learning_rate": 6.980686174774777e-05, + "loss": 0.6978650093078613, + "mean_token_accuracy": 0.7952493160963059, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7790967045511519, + "eval_loss": 0.7110744118690491, + "eval_mean_token_accuracy": 0.7964006056104388, + "eval_num_tokens": 720146.0, + "eval_runtime": 53.9404, + "eval_samples_per_second": 25.862, + "eval_steps_per_second": 3.244, + "step": 280 + }, + { + "entropy": 0.7788082852959632, + "epoch": 0.7366482504604052, + "grad_norm": 1.1519086360931396, + "learning_rate": 6.702809257595105e-05, + "loss": 0.7019347190856934, + "mean_token_accuracy": 0.7960198536515236, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7932151372092111, + "eval_loss": 0.7035816311836243, + "eval_mean_token_accuracy": 0.7979611914498466, + "eval_num_tokens": 772624.0, + "eval_runtime": 54.0517, + "eval_samples_per_second": 25.809, + "eval_steps_per_second": 3.238, + "step": 300 + }, + { + "entropy": 0.7729029789566993, + "epoch": 0.7857581338244322, + "grad_norm": 1.0247046947479248, + "learning_rate": 6.406547849821584e-05, + "loss": 0.6924194812774658, + "mean_token_accuracy": 0.7985413707792759, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7812128639221192, + "eval_loss": 0.6989337205886841, + "eval_mean_token_accuracy": 0.8005841762678964, + "eval_num_tokens": 822969.0, + "eval_runtime": 54.1306, + "eval_samples_per_second": 25.771, + "eval_steps_per_second": 3.233, + "step": 320 + }, + { + "entropy": 0.7627703189849854, + "epoch": 0.8348680171884592, + "grad_norm": 0.957506537437439, + "learning_rate": 6.0940715405257566e-05, + "loss": 0.679268217086792, + "mean_token_accuracy": 0.8040587857365609, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7624005011149815, + "eval_loss": 0.6934819221496582, + "eval_mean_token_accuracy": 0.8014607231957572, + "eval_num_tokens": 873459.0, + "eval_runtime": 54.1656, + "eval_samples_per_second": 25.754, + "eval_steps_per_second": 3.231, + "step": 340 + }, + { + "entropy": 0.763699171692133, + "epoch": 0.8839779005524862, + "grad_norm": 1.4329766035079956, + "learning_rate": 5.767668664158135e-05, + "loss": 0.6764235019683837, + "mean_token_accuracy": 0.8051542706787587, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.7163350166593279, + "eval_loss": 0.6882685422897339, + "eval_mean_token_accuracy": 0.802961312021528, + "eval_num_tokens": 925669.0, + "eval_runtime": 53.8288, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 360 + }, + { + "entropy": 0.7710359543561935, + "epoch": 0.9330877839165131, + "grad_norm": 1.141253113746643, + "learning_rate": 5.429729542559189e-05, + "loss": 0.6897628784179688, + "mean_token_accuracy": 0.8005676351487636, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7462973778588431, + "eval_loss": 0.6829195022583008, + "eval_mean_token_accuracy": 0.8036447630609784, + "eval_num_tokens": 973436.0, + "eval_runtime": 53.8276, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 380 + }, + { + "entropy": 0.7500453963875771, + "epoch": 0.9821976672805403, + "grad_norm": 0.9055600166320801, + "learning_rate": 5.0827289800936774e-05, + "loss": 0.6726306915283203, + "mean_token_accuracy": 0.8026990614831447, + "num_tokens": 1026370.0, + "step": 400 + }, + { + "epoch": 0.9821976672805403, + "eval_entropy": 0.745090776511601, + "eval_loss": 0.6795706152915955, + "eval_mean_token_accuracy": 0.8038556255613054, + "eval_num_tokens": 1026370.0, + "eval_runtime": 53.9485, + "eval_samples_per_second": 25.858, + "eval_steps_per_second": 3.244, + "step": 400 + }, + { + "entropy": 0.7288912821125675, + "epoch": 1.0294659300184161, + "grad_norm": 0.9075890779495239, + "learning_rate": 4.729208140100435e-05, + "loss": 0.6533905506134033, + "mean_token_accuracy": 0.807745551908171, + "num_tokens": 1076273.0, + "step": 420 + }, + { + "epoch": 1.0294659300184161, + "eval_entropy": 0.7279435137340001, + "eval_loss": 0.6758045554161072, + "eval_mean_token_accuracy": 0.8043134975433349, + "eval_num_tokens": 1076273.0, + "eval_runtime": 53.8173, + "eval_samples_per_second": 25.921, + "eval_steps_per_second": 3.252, + "step": 420 + }, + { + "entropy": 0.7183651916682721, + "epoch": 1.0785758133824432, + "grad_norm": 0.9271263480186462, + "learning_rate": 4.371755935380415e-05, + "loss": 0.6382484912872315, + "mean_token_accuracy": 0.810582410544157, + "num_tokens": 1133367.0, + "step": 440 + }, + { + "epoch": 1.0785758133824432, + "eval_entropy": 0.7224361307280405, + "eval_loss": 0.6700165271759033, + "eval_mean_token_accuracy": 0.8052176751409258, + "eval_num_tokens": 1133367.0, + "eval_runtime": 54.2346, + "eval_samples_per_second": 25.722, + "eval_steps_per_second": 3.227, + "step": 440 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.8141134696144256e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b0070f83a6a0f6664e390aff6d850349d04ef20a --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-460/trainer_state.json @@ -0,0 +1,517 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.1276856967464703, + "eval_steps": 20, + "global_step": 460, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + }, + { + "entropy": 0.781412948668003, + "epoch": 0.6875383670963782, + "grad_norm": 1.1252366304397583, + "learning_rate": 6.980686174774777e-05, + "loss": 0.6978650093078613, + "mean_token_accuracy": 0.7952493160963059, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7790967045511519, + "eval_loss": 0.7110744118690491, + "eval_mean_token_accuracy": 0.7964006056104388, + "eval_num_tokens": 720146.0, + "eval_runtime": 53.9404, + "eval_samples_per_second": 25.862, + "eval_steps_per_second": 3.244, + "step": 280 + }, + { + "entropy": 0.7788082852959632, + "epoch": 0.7366482504604052, + "grad_norm": 1.1519086360931396, + "learning_rate": 6.702809257595105e-05, + "loss": 0.7019347190856934, + "mean_token_accuracy": 0.7960198536515236, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7932151372092111, + "eval_loss": 0.7035816311836243, + "eval_mean_token_accuracy": 0.7979611914498466, + "eval_num_tokens": 772624.0, + "eval_runtime": 54.0517, + "eval_samples_per_second": 25.809, + "eval_steps_per_second": 3.238, + "step": 300 + }, + { + "entropy": 0.7729029789566993, + "epoch": 0.7857581338244322, + "grad_norm": 1.0247046947479248, + "learning_rate": 6.406547849821584e-05, + "loss": 0.6924194812774658, + "mean_token_accuracy": 0.7985413707792759, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7812128639221192, + "eval_loss": 0.6989337205886841, + "eval_mean_token_accuracy": 0.8005841762678964, + "eval_num_tokens": 822969.0, + "eval_runtime": 54.1306, + "eval_samples_per_second": 25.771, + "eval_steps_per_second": 3.233, + "step": 320 + }, + { + "entropy": 0.7627703189849854, + "epoch": 0.8348680171884592, + "grad_norm": 0.957506537437439, + "learning_rate": 6.0940715405257566e-05, + "loss": 0.679268217086792, + "mean_token_accuracy": 0.8040587857365609, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7624005011149815, + "eval_loss": 0.6934819221496582, + "eval_mean_token_accuracy": 0.8014607231957572, + "eval_num_tokens": 873459.0, + "eval_runtime": 54.1656, + "eval_samples_per_second": 25.754, + "eval_steps_per_second": 3.231, + "step": 340 + }, + { + "entropy": 0.763699171692133, + "epoch": 0.8839779005524862, + "grad_norm": 1.4329766035079956, + "learning_rate": 5.767668664158135e-05, + "loss": 0.6764235019683837, + "mean_token_accuracy": 0.8051542706787587, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.7163350166593279, + "eval_loss": 0.6882685422897339, + "eval_mean_token_accuracy": 0.802961312021528, + "eval_num_tokens": 925669.0, + "eval_runtime": 53.8288, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 360 + }, + { + "entropy": 0.7710359543561935, + "epoch": 0.9330877839165131, + "grad_norm": 1.141253113746643, + "learning_rate": 5.429729542559189e-05, + "loss": 0.6897628784179688, + "mean_token_accuracy": 0.8005676351487636, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7462973778588431, + "eval_loss": 0.6829195022583008, + "eval_mean_token_accuracy": 0.8036447630609784, + "eval_num_tokens": 973436.0, + "eval_runtime": 53.8276, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 380 + }, + { + "entropy": 0.7500453963875771, + "epoch": 0.9821976672805403, + "grad_norm": 0.9055600166320801, + "learning_rate": 5.0827289800936774e-05, + "loss": 0.6726306915283203, + "mean_token_accuracy": 0.8026990614831447, + "num_tokens": 1026370.0, + "step": 400 + }, + { + "epoch": 0.9821976672805403, + "eval_entropy": 0.745090776511601, + "eval_loss": 0.6795706152915955, + "eval_mean_token_accuracy": 0.8038556255613054, + "eval_num_tokens": 1026370.0, + "eval_runtime": 53.9485, + "eval_samples_per_second": 25.858, + "eval_steps_per_second": 3.244, + "step": 400 + }, + { + "entropy": 0.7288912821125675, + "epoch": 1.0294659300184161, + "grad_norm": 0.9075890779495239, + "learning_rate": 4.729208140100435e-05, + "loss": 0.6533905506134033, + "mean_token_accuracy": 0.807745551908171, + "num_tokens": 1076273.0, + "step": 420 + }, + { + "epoch": 1.0294659300184161, + "eval_entropy": 0.7279435137340001, + "eval_loss": 0.6758045554161072, + "eval_mean_token_accuracy": 0.8043134975433349, + "eval_num_tokens": 1076273.0, + "eval_runtime": 53.8173, + "eval_samples_per_second": 25.921, + "eval_steps_per_second": 3.252, + "step": 420 + }, + { + "entropy": 0.7183651916682721, + "epoch": 1.0785758133824432, + "grad_norm": 0.9271263480186462, + "learning_rate": 4.371755935380415e-05, + "loss": 0.6382484912872315, + "mean_token_accuracy": 0.810582410544157, + "num_tokens": 1133367.0, + "step": 440 + }, + { + "epoch": 1.0785758133824432, + "eval_entropy": 0.7224361307280405, + "eval_loss": 0.6700165271759033, + "eval_mean_token_accuracy": 0.8052176751409258, + "eval_num_tokens": 1133367.0, + "eval_runtime": 54.2346, + "eval_samples_per_second": 25.722, + "eval_steps_per_second": 3.227, + "step": 440 + }, + { + "entropy": 0.7097452029585838, + "epoch": 1.1276856967464703, + "grad_norm": 0.8726329207420349, + "learning_rate": 4.012990069004659e-05, + "loss": 0.6309147834777832, + "mean_token_accuracy": 0.8141021691262722, + "num_tokens": 1186585.0, + "step": 460 + }, + { + "epoch": 1.1276856967464703, + "eval_entropy": 0.7231041298593793, + "eval_loss": 0.6672903299331665, + "eval_mean_token_accuracy": 0.8079326885087149, + "eval_num_tokens": 1186585.0, + "eval_runtime": 53.7923, + "eval_samples_per_second": 25.933, + "eval_steps_per_second": 3.253, + "step": 460 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.90096949205984e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9719c9856db8e1cc1178fbbedbbeb71d9052cd46 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.02616102090367193, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..835c4c598ae10bd4b2513fe7d2d581645707082b --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1167/trainer_state.json @@ -0,0 +1,297 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1167, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.035328435897827, + "epoch": 0.1287001287001287, + "grad_norm": 3.0673413276672363, + "learning_rate": 4.96378417177596e-05, + "loss": 1.9008338928222657, + "mean_token_accuracy": 0.605284059047699, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.0828855693340302, + "epoch": 0.2574002574002574, + "grad_norm": 2.4980359077453613, + "learning_rate": 0.00010028870061343267, + "loss": 0.9762400817871094, + "mean_token_accuracy": 0.7393774157762527, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 0.9727641302347183, + "epoch": 0.3861003861003861, + "grad_norm": 1.5354057550430298, + "learning_rate": 0.00015093955950910574, + "loss": 0.8738698577880859, + "mean_token_accuracy": 0.761322969198227, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 0.9465948975086212, + "epoch": 0.5148005148005148, + "grad_norm": 1.3178573846817017, + "learning_rate": 0.00020159041840477876, + "loss": 0.854097671508789, + "mean_token_accuracy": 0.7649031269550324, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9218364179134368, + "epoch": 0.6435006435006435, + "grad_norm": 1.3482104539871216, + "learning_rate": 0.00025224127730045186, + "loss": 0.8339097595214844, + "mean_token_accuracy": 0.7713709837198257, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.92090303003788, + "epoch": 0.7722007722007722, + "grad_norm": 1.1728777885437012, + "learning_rate": 0.00030289213619612493, + "loss": 0.8308000183105468, + "mean_token_accuracy": 0.7717143112421035, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9001683223247529, + "epoch": 0.9009009009009009, + "grad_norm": 1.4331250190734863, + "learning_rate": 0.00035354299509179795, + "loss": 0.8269673919677735, + "mean_token_accuracy": 0.7711574441194534, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7020757987941664, + "eval_loss": 0.8044202923774719, + "eval_mean_token_accuracy": 0.7759947469553996, + "eval_num_tokens": 527146.0, + "eval_runtime": 78.3605, + "eval_samples_per_second": 19.729, + "eval_steps_per_second": 2.476, + "step": 389 + }, + { + "entropy": 0.8767053924425684, + "epoch": 1.0283140283140284, + "grad_norm": 1.3621660470962524, + "learning_rate": 0.0003940557495448841, + "loss": 0.8080394744873047, + "mean_token_accuracy": 0.7781236532962683, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8415434396266938, + "epoch": 1.157014157014157, + "grad_norm": 1.734777808189392, + "learning_rate": 0.00039377817338715463, + "loss": 0.7779766082763672, + "mean_token_accuracy": 0.7830026543140411, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8639528834819794, + "epoch": 1.2857142857142856, + "grad_norm": 1.3795595169067383, + "learning_rate": 0.00039310460257672286, + "loss": 0.7801459503173828, + "mean_token_accuracy": 0.7819079357385635, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.8538346707820892, + "epoch": 1.4144144144144144, + "grad_norm": 1.6661036014556885, + "learning_rate": 0.00039203639282089, + "loss": 0.7902972412109375, + "mean_token_accuracy": 0.7807946759462356, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8081058472394943, + "epoch": 1.5431145431145432, + "grad_norm": 1.2289175987243652, + "learning_rate": 0.00039057569412340117, + "loss": 0.7647433471679688, + "mean_token_accuracy": 0.7846344596147538, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8334913140535355, + "epoch": 1.6718146718146718, + "grad_norm": 2.2659049034118652, + "learning_rate": 0.0003887254464570972, + "loss": 0.7828218078613282, + "mean_token_accuracy": 0.7819693273305893, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8104432183504104, + "epoch": 1.8005148005148004, + "grad_norm": 1.7065047025680542, + "learning_rate": 0.00038648937384658125, + "loss": 0.763125, + "mean_token_accuracy": 0.7861280244588852, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8062294274568558, + "epoch": 1.9292149292149292, + "grad_norm": 1.3628491163253784, + "learning_rate": 0.0003838719768728124, + "loss": 0.7553546905517579, + "mean_token_accuracy": 0.7892439645528794, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.679631345665332, + "eval_loss": 0.7782689929008484, + "eval_mean_token_accuracy": 0.787303302398662, + "eval_num_tokens": 1054292.0, + "eval_runtime": 75.9942, + "eval_samples_per_second": 20.344, + "eval_steps_per_second": 2.553, + "step": 778 + }, + { + "entropy": 0.749360041786926, + "epoch": 2.056628056628057, + "grad_norm": 1.084079623222351, + "learning_rate": 0.0003808785236147101, + "loss": 0.7160065460205078, + "mean_token_accuracy": 0.7958323576233604, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.6975917857885361, + "epoch": 2.1853281853281854, + "grad_norm": 1.3939169645309448, + "learning_rate": 0.00037751503904600377, + "loss": 0.6710267639160157, + "mean_token_accuracy": 0.8034947836399078, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7072162652015686, + "epoch": 2.314028314028314, + "grad_norm": 1.9786381721496582, + "learning_rate": 0.0003737882929086661, + "loss": 0.6777350616455078, + "mean_token_accuracy": 0.8026440101861954, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7069408267736434, + "epoch": 2.4427284427284426, + "grad_norm": 1.3146620988845825, + "learning_rate": 0.00036970578608734, + "loss": 0.6795073699951172, + "mean_token_accuracy": 0.8013493180274963, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.6993900266289711, + "epoch": 2.571428571428571, + "grad_norm": 1.5064544677734375, + "learning_rate": 0.00036527573551218173, + "loss": 0.6791993713378907, + "mean_token_accuracy": 0.8022604930400848, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.6983578675985336, + "epoch": 2.7001287001287, + "grad_norm": 1.2354060411453247, + "learning_rate": 0.0003605070576205073, + "loss": 0.6693215179443359, + "mean_token_accuracy": 0.8064261078834534, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7105095881223679, + "epoch": 2.828828828828829, + "grad_norm": 1.182181477546692, + "learning_rate": 0.00035540935041052915, + "loss": 0.6762094116210937, + "mean_token_accuracy": 0.8052380502223968, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7148650509119033, + "epoch": 2.9575289575289574, + "grad_norm": 0.9586004018783569, + "learning_rate": 0.00034999287412330383, + "loss": 0.675987777709961, + "mean_token_accuracy": 0.8049957078695297, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6378419605112567, + "eval_loss": 0.7481877207756042, + "eval_mean_token_accuracy": 0.7935938085477376, + "eval_num_tokens": 1581438.0, + "eval_runtime": 75.8368, + "eval_samples_per_second": 20.386, + "eval_steps_per_second": 2.558, + "step": 1167 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.584625602801459e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9719c9856db8e1cc1178fbbedbbeb71d9052cd46 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.02616102090367193, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e35ab947a237cd2593f381729b41bdeb94098dca --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1556/trainer_state.json @@ -0,0 +1,388 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1556, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.035328435897827, + "epoch": 0.1287001287001287, + "grad_norm": 3.0673413276672363, + "learning_rate": 4.96378417177596e-05, + "loss": 1.9008338928222657, + "mean_token_accuracy": 0.605284059047699, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.0828855693340302, + "epoch": 0.2574002574002574, + "grad_norm": 2.4980359077453613, + "learning_rate": 0.00010028870061343267, + "loss": 0.9762400817871094, + "mean_token_accuracy": 0.7393774157762527, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 0.9727641302347183, + "epoch": 0.3861003861003861, + "grad_norm": 1.5354057550430298, + "learning_rate": 0.00015093955950910574, + "loss": 0.8738698577880859, + "mean_token_accuracy": 0.761322969198227, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 0.9465948975086212, + "epoch": 0.5148005148005148, + "grad_norm": 1.3178573846817017, + "learning_rate": 0.00020159041840477876, + "loss": 0.854097671508789, + "mean_token_accuracy": 0.7649031269550324, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9218364179134368, + "epoch": 0.6435006435006435, + "grad_norm": 1.3482104539871216, + "learning_rate": 0.00025224127730045186, + "loss": 0.8339097595214844, + "mean_token_accuracy": 0.7713709837198257, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.92090303003788, + "epoch": 0.7722007722007722, + "grad_norm": 1.1728777885437012, + "learning_rate": 0.00030289213619612493, + "loss": 0.8308000183105468, + "mean_token_accuracy": 0.7717143112421035, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9001683223247529, + "epoch": 0.9009009009009009, + "grad_norm": 1.4331250190734863, + "learning_rate": 0.00035354299509179795, + "loss": 0.8269673919677735, + "mean_token_accuracy": 0.7711574441194534, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7020757987941664, + "eval_loss": 0.8044202923774719, + "eval_mean_token_accuracy": 0.7759947469553996, + "eval_num_tokens": 527146.0, + "eval_runtime": 78.3605, + "eval_samples_per_second": 19.729, + "eval_steps_per_second": 2.476, + "step": 389 + }, + { + "entropy": 0.8767053924425684, + "epoch": 1.0283140283140284, + "grad_norm": 1.3621660470962524, + "learning_rate": 0.0003940557495448841, + "loss": 0.8080394744873047, + "mean_token_accuracy": 0.7781236532962683, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8415434396266938, + "epoch": 1.157014157014157, + "grad_norm": 1.734777808189392, + "learning_rate": 0.00039377817338715463, + "loss": 0.7779766082763672, + "mean_token_accuracy": 0.7830026543140411, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8639528834819794, + "epoch": 1.2857142857142856, + "grad_norm": 1.3795595169067383, + "learning_rate": 0.00039310460257672286, + "loss": 0.7801459503173828, + "mean_token_accuracy": 0.7819079357385635, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.8538346707820892, + "epoch": 1.4144144144144144, + "grad_norm": 1.6661036014556885, + "learning_rate": 0.00039203639282089, + "loss": 0.7902972412109375, + "mean_token_accuracy": 0.7807946759462356, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8081058472394943, + "epoch": 1.5431145431145432, + "grad_norm": 1.2289175987243652, + "learning_rate": 0.00039057569412340117, + "loss": 0.7647433471679688, + "mean_token_accuracy": 0.7846344596147538, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8334913140535355, + "epoch": 1.6718146718146718, + "grad_norm": 2.2659049034118652, + "learning_rate": 0.0003887254464570972, + "loss": 0.7828218078613282, + "mean_token_accuracy": 0.7819693273305893, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8104432183504104, + "epoch": 1.8005148005148004, + "grad_norm": 1.7065047025680542, + "learning_rate": 0.00038648937384658125, + "loss": 0.763125, + "mean_token_accuracy": 0.7861280244588852, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8062294274568558, + "epoch": 1.9292149292149292, + "grad_norm": 1.3628491163253784, + "learning_rate": 0.0003838719768728124, + "loss": 0.7553546905517579, + "mean_token_accuracy": 0.7892439645528794, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.679631345665332, + "eval_loss": 0.7782689929008484, + "eval_mean_token_accuracy": 0.787303302398662, + "eval_num_tokens": 1054292.0, + "eval_runtime": 75.9942, + "eval_samples_per_second": 20.344, + "eval_steps_per_second": 2.553, + "step": 778 + }, + { + "entropy": 0.749360041786926, + "epoch": 2.056628056628057, + "grad_norm": 1.084079623222351, + "learning_rate": 0.0003808785236147101, + "loss": 0.7160065460205078, + "mean_token_accuracy": 0.7958323576233604, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.6975917857885361, + "epoch": 2.1853281853281854, + "grad_norm": 1.3939169645309448, + "learning_rate": 0.00037751503904600377, + "loss": 0.6710267639160157, + "mean_token_accuracy": 0.8034947836399078, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7072162652015686, + "epoch": 2.314028314028314, + "grad_norm": 1.9786381721496582, + "learning_rate": 0.0003737882929086661, + "loss": 0.6777350616455078, + "mean_token_accuracy": 0.8026440101861954, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7069408267736434, + "epoch": 2.4427284427284426, + "grad_norm": 1.3146620988845825, + "learning_rate": 0.00036970578608734, + "loss": 0.6795073699951172, + "mean_token_accuracy": 0.8013493180274963, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.6993900266289711, + "epoch": 2.571428571428571, + "grad_norm": 1.5064544677734375, + "learning_rate": 0.00036527573551218173, + "loss": 0.6791993713378907, + "mean_token_accuracy": 0.8022604930400848, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.6983578675985336, + "epoch": 2.7001287001287, + "grad_norm": 1.2354060411453247, + "learning_rate": 0.0003605070576205073, + "loss": 0.6693215179443359, + "mean_token_accuracy": 0.8064261078834534, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7105095881223679, + "epoch": 2.828828828828829, + "grad_norm": 1.182181477546692, + "learning_rate": 0.00035540935041052915, + "loss": 0.6762094116210937, + "mean_token_accuracy": 0.8052380502223968, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7148650509119033, + "epoch": 2.9575289575289574, + "grad_norm": 0.9586004018783569, + "learning_rate": 0.00034999287412330383, + "loss": 0.675987777709961, + "mean_token_accuracy": 0.8049957078695297, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6378419605112567, + "eval_loss": 0.7481877207756042, + "eval_mean_token_accuracy": 0.7935938085477376, + "eval_num_tokens": 1581438.0, + "eval_runtime": 75.8368, + "eval_samples_per_second": 20.386, + "eval_steps_per_second": 2.558, + "step": 1167 + }, + { + "entropy": 0.6485739145616088, + "epoch": 3.0849420849420848, + "grad_norm": 1.1198654174804688, + "learning_rate": 0.00034426853059177186, + "loss": 0.6087686157226563, + "mean_token_accuracy": 0.8164294560750326, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6030477187037468, + "epoch": 3.213642213642214, + "grad_norm": 1.1516684293746948, + "learning_rate": 0.00033824784129845556, + "loss": 0.5574645614624023, + "mean_token_accuracy": 0.8271971315145492, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6140900093317032, + "epoch": 3.3423423423423424, + "grad_norm": 1.372568964958191, + "learning_rate": 0.00033194292418597715, + "loss": 0.5769831848144531, + "mean_token_accuracy": 0.8246584522724152, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6223019778728485, + "epoch": 3.471042471042471, + "grad_norm": 1.5307430028915405, + "learning_rate": 0.0003253664692670717, + "loss": 0.5899515914916992, + "mean_token_accuracy": 0.820688226222992, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6215043878555297, + "epoch": 3.5997425997425996, + "grad_norm": 1.487574577331543, + "learning_rate": 0.0003185317130831854, + "loss": 0.5944060134887695, + "mean_token_accuracy": 0.8215733963251114, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6398005878925324, + "epoch": 3.7284427284427286, + "grad_norm": 1.054810881614685, + "learning_rate": 0.0003114524120630655, + "loss": 0.5942705154418946, + "mean_token_accuracy": 0.8215270626544953, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6384156176447868, + "epoch": 3.857142857142857, + "grad_norm": 1.5161454677581787, + "learning_rate": 0.00030414281483496505, + "loss": 0.5947936630249023, + "mean_token_accuracy": 0.8206489557027816, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.598966498374939, + "epoch": 3.985842985842986, + "grad_norm": 1.2136448621749878, + "learning_rate": 0.00029661763354818884, + "loss": 0.5746045684814454, + "mean_token_accuracy": 0.8274296247959136, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5822035986738107, + "eval_loss": 0.7569039463996887, + "eval_mean_token_accuracy": 0.7907421751735136, + "eval_num_tokens": 2108584.0, + "eval_runtime": 76.165, + "eval_samples_per_second": 20.298, + "eval_steps_per_second": 2.547, + "step": 1556 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.447669114485645e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9719c9856db8e1cc1178fbbedbbeb71d9052cd46 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.02616102090367193, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..eec1f6c79bd3011c0e43eb58ab6dc5eae0749bf8 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1945/trainer_state.json @@ -0,0 +1,469 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 1945, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.035328435897827, + "epoch": 0.1287001287001287, + "grad_norm": 3.0673413276672363, + "learning_rate": 4.96378417177596e-05, + "loss": 1.9008338928222657, + "mean_token_accuracy": 0.605284059047699, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.0828855693340302, + "epoch": 0.2574002574002574, + "grad_norm": 2.4980359077453613, + "learning_rate": 0.00010028870061343267, + "loss": 0.9762400817871094, + "mean_token_accuracy": 0.7393774157762527, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 0.9727641302347183, + "epoch": 0.3861003861003861, + "grad_norm": 1.5354057550430298, + "learning_rate": 0.00015093955950910574, + "loss": 0.8738698577880859, + "mean_token_accuracy": 0.761322969198227, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 0.9465948975086212, + "epoch": 0.5148005148005148, + "grad_norm": 1.3178573846817017, + "learning_rate": 0.00020159041840477876, + "loss": 0.854097671508789, + "mean_token_accuracy": 0.7649031269550324, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9218364179134368, + "epoch": 0.6435006435006435, + "grad_norm": 1.3482104539871216, + "learning_rate": 0.00025224127730045186, + "loss": 0.8339097595214844, + "mean_token_accuracy": 0.7713709837198257, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.92090303003788, + "epoch": 0.7722007722007722, + "grad_norm": 1.1728777885437012, + "learning_rate": 0.00030289213619612493, + "loss": 0.8308000183105468, + "mean_token_accuracy": 0.7717143112421035, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9001683223247529, + "epoch": 0.9009009009009009, + "grad_norm": 1.4331250190734863, + "learning_rate": 0.00035354299509179795, + "loss": 0.8269673919677735, + "mean_token_accuracy": 0.7711574441194534, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7020757987941664, + "eval_loss": 0.8044202923774719, + "eval_mean_token_accuracy": 0.7759947469553996, + "eval_num_tokens": 527146.0, + "eval_runtime": 78.3605, + "eval_samples_per_second": 19.729, + "eval_steps_per_second": 2.476, + "step": 389 + }, + { + "entropy": 0.8767053924425684, + "epoch": 1.0283140283140284, + "grad_norm": 1.3621660470962524, + "learning_rate": 0.0003940557495448841, + "loss": 0.8080394744873047, + "mean_token_accuracy": 0.7781236532962683, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8415434396266938, + "epoch": 1.157014157014157, + "grad_norm": 1.734777808189392, + "learning_rate": 0.00039377817338715463, + "loss": 0.7779766082763672, + "mean_token_accuracy": 0.7830026543140411, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8639528834819794, + "epoch": 1.2857142857142856, + "grad_norm": 1.3795595169067383, + "learning_rate": 0.00039310460257672286, + "loss": 0.7801459503173828, + "mean_token_accuracy": 0.7819079357385635, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.8538346707820892, + "epoch": 1.4144144144144144, + "grad_norm": 1.6661036014556885, + "learning_rate": 0.00039203639282089, + "loss": 0.7902972412109375, + "mean_token_accuracy": 0.7807946759462356, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8081058472394943, + "epoch": 1.5431145431145432, + "grad_norm": 1.2289175987243652, + "learning_rate": 0.00039057569412340117, + "loss": 0.7647433471679688, + "mean_token_accuracy": 0.7846344596147538, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8334913140535355, + "epoch": 1.6718146718146718, + "grad_norm": 2.2659049034118652, + "learning_rate": 0.0003887254464570972, + "loss": 0.7828218078613282, + "mean_token_accuracy": 0.7819693273305893, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8104432183504104, + "epoch": 1.8005148005148004, + "grad_norm": 1.7065047025680542, + "learning_rate": 0.00038648937384658125, + "loss": 0.763125, + "mean_token_accuracy": 0.7861280244588852, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8062294274568558, + "epoch": 1.9292149292149292, + "grad_norm": 1.3628491163253784, + "learning_rate": 0.0003838719768728124, + "loss": 0.7553546905517579, + "mean_token_accuracy": 0.7892439645528794, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.679631345665332, + "eval_loss": 0.7782689929008484, + "eval_mean_token_accuracy": 0.787303302398662, + "eval_num_tokens": 1054292.0, + "eval_runtime": 75.9942, + "eval_samples_per_second": 20.344, + "eval_steps_per_second": 2.553, + "step": 778 + }, + { + "entropy": 0.749360041786926, + "epoch": 2.056628056628057, + "grad_norm": 1.084079623222351, + "learning_rate": 0.0003808785236147101, + "loss": 0.7160065460205078, + "mean_token_accuracy": 0.7958323576233604, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.6975917857885361, + "epoch": 2.1853281853281854, + "grad_norm": 1.3939169645309448, + "learning_rate": 0.00037751503904600377, + "loss": 0.6710267639160157, + "mean_token_accuracy": 0.8034947836399078, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7072162652015686, + "epoch": 2.314028314028314, + "grad_norm": 1.9786381721496582, + "learning_rate": 0.0003737882929086661, + "loss": 0.6777350616455078, + "mean_token_accuracy": 0.8026440101861954, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7069408267736434, + "epoch": 2.4427284427284426, + "grad_norm": 1.3146620988845825, + "learning_rate": 0.00036970578608734, + "loss": 0.6795073699951172, + "mean_token_accuracy": 0.8013493180274963, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.6993900266289711, + "epoch": 2.571428571428571, + "grad_norm": 1.5064544677734375, + "learning_rate": 0.00036527573551218173, + "loss": 0.6791993713378907, + "mean_token_accuracy": 0.8022604930400848, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.6983578675985336, + "epoch": 2.7001287001287, + "grad_norm": 1.2354060411453247, + "learning_rate": 0.0003605070576205073, + "loss": 0.6693215179443359, + "mean_token_accuracy": 0.8064261078834534, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7105095881223679, + "epoch": 2.828828828828829, + "grad_norm": 1.182181477546692, + "learning_rate": 0.00035540935041052915, + "loss": 0.6762094116210937, + "mean_token_accuracy": 0.8052380502223968, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7148650509119033, + "epoch": 2.9575289575289574, + "grad_norm": 0.9586004018783569, + "learning_rate": 0.00034999287412330383, + "loss": 0.675987777709961, + "mean_token_accuracy": 0.8049957078695297, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6378419605112567, + "eval_loss": 0.7481877207756042, + "eval_mean_token_accuracy": 0.7935938085477376, + "eval_num_tokens": 1581438.0, + "eval_runtime": 75.8368, + "eval_samples_per_second": 20.386, + "eval_steps_per_second": 2.558, + "step": 1167 + }, + { + "entropy": 0.6485739145616088, + "epoch": 3.0849420849420848, + "grad_norm": 1.1198654174804688, + "learning_rate": 0.00034426853059177186, + "loss": 0.6087686157226563, + "mean_token_accuracy": 0.8164294560750326, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6030477187037468, + "epoch": 3.213642213642214, + "grad_norm": 1.1516684293746948, + "learning_rate": 0.00033824784129845556, + "loss": 0.5574645614624023, + "mean_token_accuracy": 0.8271971315145492, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6140900093317032, + "epoch": 3.3423423423423424, + "grad_norm": 1.372568964958191, + "learning_rate": 0.00033194292418597715, + "loss": 0.5769831848144531, + "mean_token_accuracy": 0.8246584522724152, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6223019778728485, + "epoch": 3.471042471042471, + "grad_norm": 1.5307430028915405, + "learning_rate": 0.0003253664692670717, + "loss": 0.5899515914916992, + "mean_token_accuracy": 0.820688226222992, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6215043878555297, + "epoch": 3.5997425997425996, + "grad_norm": 1.487574577331543, + "learning_rate": 0.0003185317130831854, + "loss": 0.5944060134887695, + "mean_token_accuracy": 0.8215733963251114, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6398005878925324, + "epoch": 3.7284427284427286, + "grad_norm": 1.054810881614685, + "learning_rate": 0.0003114524120630655, + "loss": 0.5942705154418946, + "mean_token_accuracy": 0.8215270626544953, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6384156176447868, + "epoch": 3.857142857142857, + "grad_norm": 1.5161454677581787, + "learning_rate": 0.00030414281483496505, + "loss": 0.5947936630249023, + "mean_token_accuracy": 0.8206489557027816, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.598966498374939, + "epoch": 3.985842985842986, + "grad_norm": 1.2136448621749878, + "learning_rate": 0.00029661763354818884, + "loss": 0.5746045684814454, + "mean_token_accuracy": 0.8274296247959136, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5822035986738107, + "eval_loss": 0.7569039463996887, + "eval_mean_token_accuracy": 0.7907421751735136, + "eval_num_tokens": 2108584.0, + "eval_runtime": 76.165, + "eval_samples_per_second": 20.298, + "eval_steps_per_second": 2.547, + "step": 1556 + }, + { + "entropy": 0.5126810152121265, + "epoch": 4.113256113256114, + "grad_norm": 1.2219198942184448, + "learning_rate": 0.00028889201426170327, + "loss": 0.4712484359741211, + "mean_token_accuracy": 0.8507941756585632, + "num_tokens": 2165782.0, + "step": 1600 + }, + { + "entropy": 0.5017523977160454, + "epoch": 4.241956241956242, + "grad_norm": 1.1246048212051392, + "learning_rate": 0.00028098150645940856, + "loss": 0.45758853912353514, + "mean_token_accuracy": 0.8549614226818085, + "num_tokens": 2233206.0, + "step": 1650 + }, + { + "entropy": 0.5128893256187439, + "epoch": 4.370656370656371, + "grad_norm": 1.2206052541732788, + "learning_rate": 0.000272902031753431, + "loss": 0.4767184066772461, + "mean_token_accuracy": 0.8486203062534332, + "num_tokens": 2301399.0, + "step": 1700 + }, + { + "entropy": 0.500448043346405, + "epoch": 4.499356499356499, + "grad_norm": 1.099212884902954, + "learning_rate": 0.0002646698518384262, + "loss": 0.46724525451660154, + "mean_token_accuracy": 0.8502935725450516, + "num_tokens": 2370831.0, + "step": 1750 + }, + { + "entropy": 0.5091731676459312, + "epoch": 4.628056628056628, + "grad_norm": 1.0495911836624146, + "learning_rate": 0.0002563015357613929, + "loss": 0.4759058380126953, + "mean_token_accuracy": 0.8488407224416733, + "num_tokens": 2439168.0, + "step": 1800 + }, + { + "entropy": 0.515411552786827, + "epoch": 4.756756756756757, + "grad_norm": 0.9209436774253845, + "learning_rate": 0.00024781392657287327, + "loss": 0.48281356811523435, + "mean_token_accuracy": 0.8456312155723572, + "num_tokens": 2508036.0, + "step": 1850 + }, + { + "entropy": 0.5016480022668839, + "epoch": 4.885456885456885, + "grad_norm": 1.5305742025375366, + "learning_rate": 0.00023922410742666182, + "loss": 0.4707856369018555, + "mean_token_accuracy": 0.8486748856306076, + "num_tokens": 2577025.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5264220541899967, + "eval_loss": 0.7557247281074524, + "eval_mean_token_accuracy": 0.7990376009154565, + "eval_num_tokens": 2635730.0, + "eval_runtime": 76.2827, + "eval_samples_per_second": 20.267, + "eval_steps_per_second": 2.543, + "step": 1945 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.308754736195443e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9719c9856db8e1cc1178fbbedbbeb71d9052cd46 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.02616102090367193, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1ae4e5b4808df3649ae9abfc4f918cd16ca9ed93 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2334/trainer_state.json @@ -0,0 +1,560 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2334, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.035328435897827, + "epoch": 0.1287001287001287, + "grad_norm": 3.0673413276672363, + "learning_rate": 4.96378417177596e-05, + "loss": 1.9008338928222657, + "mean_token_accuracy": 0.605284059047699, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.0828855693340302, + "epoch": 0.2574002574002574, + "grad_norm": 2.4980359077453613, + "learning_rate": 0.00010028870061343267, + "loss": 0.9762400817871094, + "mean_token_accuracy": 0.7393774157762527, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 0.9727641302347183, + "epoch": 0.3861003861003861, + "grad_norm": 1.5354057550430298, + "learning_rate": 0.00015093955950910574, + "loss": 0.8738698577880859, + "mean_token_accuracy": 0.761322969198227, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 0.9465948975086212, + "epoch": 0.5148005148005148, + "grad_norm": 1.3178573846817017, + "learning_rate": 0.00020159041840477876, + "loss": 0.854097671508789, + "mean_token_accuracy": 0.7649031269550324, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9218364179134368, + "epoch": 0.6435006435006435, + "grad_norm": 1.3482104539871216, + "learning_rate": 0.00025224127730045186, + "loss": 0.8339097595214844, + "mean_token_accuracy": 0.7713709837198257, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.92090303003788, + "epoch": 0.7722007722007722, + "grad_norm": 1.1728777885437012, + "learning_rate": 0.00030289213619612493, + "loss": 0.8308000183105468, + "mean_token_accuracy": 0.7717143112421035, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9001683223247529, + "epoch": 0.9009009009009009, + "grad_norm": 1.4331250190734863, + "learning_rate": 0.00035354299509179795, + "loss": 0.8269673919677735, + "mean_token_accuracy": 0.7711574441194534, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7020757987941664, + "eval_loss": 0.8044202923774719, + "eval_mean_token_accuracy": 0.7759947469553996, + "eval_num_tokens": 527146.0, + "eval_runtime": 78.3605, + "eval_samples_per_second": 19.729, + "eval_steps_per_second": 2.476, + "step": 389 + }, + { + "entropy": 0.8767053924425684, + "epoch": 1.0283140283140284, + "grad_norm": 1.3621660470962524, + "learning_rate": 0.0003940557495448841, + "loss": 0.8080394744873047, + "mean_token_accuracy": 0.7781236532962683, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8415434396266938, + "epoch": 1.157014157014157, + "grad_norm": 1.734777808189392, + "learning_rate": 0.00039377817338715463, + "loss": 0.7779766082763672, + "mean_token_accuracy": 0.7830026543140411, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8639528834819794, + "epoch": 1.2857142857142856, + "grad_norm": 1.3795595169067383, + "learning_rate": 0.00039310460257672286, + "loss": 0.7801459503173828, + "mean_token_accuracy": 0.7819079357385635, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.8538346707820892, + "epoch": 1.4144144144144144, + "grad_norm": 1.6661036014556885, + "learning_rate": 0.00039203639282089, + "loss": 0.7902972412109375, + "mean_token_accuracy": 0.7807946759462356, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8081058472394943, + "epoch": 1.5431145431145432, + "grad_norm": 1.2289175987243652, + "learning_rate": 0.00039057569412340117, + "loss": 0.7647433471679688, + "mean_token_accuracy": 0.7846344596147538, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8334913140535355, + "epoch": 1.6718146718146718, + "grad_norm": 2.2659049034118652, + "learning_rate": 0.0003887254464570972, + "loss": 0.7828218078613282, + "mean_token_accuracy": 0.7819693273305893, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8104432183504104, + "epoch": 1.8005148005148004, + "grad_norm": 1.7065047025680542, + "learning_rate": 0.00038648937384658125, + "loss": 0.763125, + "mean_token_accuracy": 0.7861280244588852, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8062294274568558, + "epoch": 1.9292149292149292, + "grad_norm": 1.3628491163253784, + "learning_rate": 0.0003838719768728124, + "loss": 0.7553546905517579, + "mean_token_accuracy": 0.7892439645528794, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.679631345665332, + "eval_loss": 0.7782689929008484, + "eval_mean_token_accuracy": 0.787303302398662, + "eval_num_tokens": 1054292.0, + "eval_runtime": 75.9942, + "eval_samples_per_second": 20.344, + "eval_steps_per_second": 2.553, + "step": 778 + }, + { + "entropy": 0.749360041786926, + "epoch": 2.056628056628057, + "grad_norm": 1.084079623222351, + "learning_rate": 0.0003808785236147101, + "loss": 0.7160065460205078, + "mean_token_accuracy": 0.7958323576233604, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.6975917857885361, + "epoch": 2.1853281853281854, + "grad_norm": 1.3939169645309448, + "learning_rate": 0.00037751503904600377, + "loss": 0.6710267639160157, + "mean_token_accuracy": 0.8034947836399078, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7072162652015686, + "epoch": 2.314028314028314, + "grad_norm": 1.9786381721496582, + "learning_rate": 0.0003737882929086661, + "loss": 0.6777350616455078, + "mean_token_accuracy": 0.8026440101861954, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7069408267736434, + "epoch": 2.4427284427284426, + "grad_norm": 1.3146620988845825, + "learning_rate": 0.00036970578608734, + "loss": 0.6795073699951172, + "mean_token_accuracy": 0.8013493180274963, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.6993900266289711, + "epoch": 2.571428571428571, + "grad_norm": 1.5064544677734375, + "learning_rate": 0.00036527573551218173, + "loss": 0.6791993713378907, + "mean_token_accuracy": 0.8022604930400848, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.6983578675985336, + "epoch": 2.7001287001287, + "grad_norm": 1.2354060411453247, + "learning_rate": 0.0003605070576205073, + "loss": 0.6693215179443359, + "mean_token_accuracy": 0.8064261078834534, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7105095881223679, + "epoch": 2.828828828828829, + "grad_norm": 1.182181477546692, + "learning_rate": 0.00035540935041052915, + "loss": 0.6762094116210937, + "mean_token_accuracy": 0.8052380502223968, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7148650509119033, + "epoch": 2.9575289575289574, + "grad_norm": 0.9586004018783569, + "learning_rate": 0.00034999287412330383, + "loss": 0.675987777709961, + "mean_token_accuracy": 0.8049957078695297, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6378419605112567, + "eval_loss": 0.7481877207756042, + "eval_mean_token_accuracy": 0.7935938085477376, + "eval_num_tokens": 1581438.0, + "eval_runtime": 75.8368, + "eval_samples_per_second": 20.386, + "eval_steps_per_second": 2.558, + "step": 1167 + }, + { + "entropy": 0.6485739145616088, + "epoch": 3.0849420849420848, + "grad_norm": 1.1198654174804688, + "learning_rate": 0.00034426853059177186, + "loss": 0.6087686157226563, + "mean_token_accuracy": 0.8164294560750326, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6030477187037468, + "epoch": 3.213642213642214, + "grad_norm": 1.1516684293746948, + "learning_rate": 0.00033824784129845556, + "loss": 0.5574645614624023, + "mean_token_accuracy": 0.8271971315145492, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6140900093317032, + "epoch": 3.3423423423423424, + "grad_norm": 1.372568964958191, + "learning_rate": 0.00033194292418597715, + "loss": 0.5769831848144531, + "mean_token_accuracy": 0.8246584522724152, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6223019778728485, + "epoch": 3.471042471042471, + "grad_norm": 1.5307430028915405, + "learning_rate": 0.0003253664692670717, + "loss": 0.5899515914916992, + "mean_token_accuracy": 0.820688226222992, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6215043878555297, + "epoch": 3.5997425997425996, + "grad_norm": 1.487574577331543, + "learning_rate": 0.0003185317130831854, + "loss": 0.5944060134887695, + "mean_token_accuracy": 0.8215733963251114, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6398005878925324, + "epoch": 3.7284427284427286, + "grad_norm": 1.054810881614685, + "learning_rate": 0.0003114524120630655, + "loss": 0.5942705154418946, + "mean_token_accuracy": 0.8215270626544953, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6384156176447868, + "epoch": 3.857142857142857, + "grad_norm": 1.5161454677581787, + "learning_rate": 0.00030414281483496505, + "loss": 0.5947936630249023, + "mean_token_accuracy": 0.8206489557027816, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.598966498374939, + "epoch": 3.985842985842986, + "grad_norm": 1.2136448621749878, + "learning_rate": 0.00029661763354818884, + "loss": 0.5746045684814454, + "mean_token_accuracy": 0.8274296247959136, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5822035986738107, + "eval_loss": 0.7569039463996887, + "eval_mean_token_accuracy": 0.7907421751735136, + "eval_num_tokens": 2108584.0, + "eval_runtime": 76.165, + "eval_samples_per_second": 20.298, + "eval_steps_per_second": 2.547, + "step": 1556 + }, + { + "entropy": 0.5126810152121265, + "epoch": 4.113256113256114, + "grad_norm": 1.2219198942184448, + "learning_rate": 0.00028889201426170327, + "loss": 0.4712484359741211, + "mean_token_accuracy": 0.8507941756585632, + "num_tokens": 2165782.0, + "step": 1600 + }, + { + "entropy": 0.5017523977160454, + "epoch": 4.241956241956242, + "grad_norm": 1.1246048212051392, + "learning_rate": 0.00028098150645940856, + "loss": 0.45758853912353514, + "mean_token_accuracy": 0.8549614226818085, + "num_tokens": 2233206.0, + "step": 1650 + }, + { + "entropy": 0.5128893256187439, + "epoch": 4.370656370656371, + "grad_norm": 1.2206052541732788, + "learning_rate": 0.000272902031753431, + "loss": 0.4767184066772461, + "mean_token_accuracy": 0.8486203062534332, + "num_tokens": 2301399.0, + "step": 1700 + }, + { + "entropy": 0.500448043346405, + "epoch": 4.499356499356499, + "grad_norm": 1.099212884902954, + "learning_rate": 0.0002646698518384262, + "loss": 0.46724525451660154, + "mean_token_accuracy": 0.8502935725450516, + "num_tokens": 2370831.0, + "step": 1750 + }, + { + "entropy": 0.5091731676459312, + "epoch": 4.628056628056628, + "grad_norm": 1.0495911836624146, + "learning_rate": 0.0002563015357613929, + "loss": 0.4759058380126953, + "mean_token_accuracy": 0.8488407224416733, + "num_tokens": 2439168.0, + "step": 1800 + }, + { + "entropy": 0.515411552786827, + "epoch": 4.756756756756757, + "grad_norm": 0.9209436774253845, + "learning_rate": 0.00024781392657287327, + "loss": 0.48281356811523435, + "mean_token_accuracy": 0.8456312155723572, + "num_tokens": 2508036.0, + "step": 1850 + }, + { + "entropy": 0.5016480022668839, + "epoch": 4.885456885456885, + "grad_norm": 1.5305742025375366, + "learning_rate": 0.00023922410742666182, + "loss": 0.4707856369018555, + "mean_token_accuracy": 0.8486748856306076, + "num_tokens": 2577025.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5264220541899967, + "eval_loss": 0.7557247281074524, + "eval_mean_token_accuracy": 0.7990376009154565, + "eval_num_tokens": 2635730.0, + "eval_runtime": 76.2827, + "eval_samples_per_second": 20.267, + "eval_steps_per_second": 2.543, + "step": 1945 + }, + { + "entropy": 0.5138596534126937, + "epoch": 5.012870012870013, + "grad_norm": 0.7728239893913269, + "learning_rate": 0.00023054936719625412, + "loss": 0.4731882095336914, + "mean_token_accuracy": 0.8493862681918674, + "num_tokens": 2642803.0, + "step": 1950 + }, + { + "entropy": 0.3956746557354927, + "epoch": 5.141570141570142, + "grad_norm": 0.852727472782135, + "learning_rate": 0.00022180716567724064, + "loss": 0.3441731262207031, + "mean_token_accuracy": 0.8864887124300003, + "num_tokens": 2708703.0, + "step": 2000 + }, + { + "entropy": 0.3931361359357834, + "epoch": 5.27027027027027, + "grad_norm": 1.062345266342163, + "learning_rate": 0.00021301509844568218, + "loss": 0.34529010772705077, + "mean_token_accuracy": 0.88379523396492, + "num_tokens": 2777956.0, + "step": 2050 + }, + { + "entropy": 0.3996828556060791, + "epoch": 5.398970398970399, + "grad_norm": 1.1475750207901, + "learning_rate": 0.00020419086144319811, + "loss": 0.35055686950683596, + "mean_token_accuracy": 0.8822557419538498, + "num_tokens": 2847032.0, + "step": 2100 + }, + { + "entropy": 0.3957197970151901, + "epoch": 5.527670527670527, + "grad_norm": 0.9748324751853943, + "learning_rate": 0.00019535221536004678, + "loss": 0.3523390197753906, + "mean_token_accuracy": 0.8818045473098755, + "num_tokens": 2915746.0, + "step": 2150 + }, + { + "entropy": 0.39835822969675067, + "epoch": 5.656370656370656, + "grad_norm": 1.1422828435897827, + "learning_rate": 0.00018651694988788578, + "loss": 0.35444053649902346, + "mean_token_accuracy": 0.8799562251567841, + "num_tokens": 2984346.0, + "step": 2200 + }, + { + "entropy": 0.40228330612182617, + "epoch": 5.785070785070785, + "grad_norm": 0.9757609963417053, + "learning_rate": 0.0001777028479141599, + "loss": 0.3603257751464844, + "mean_token_accuracy": 0.8790501236915589, + "num_tokens": 3051863.0, + "step": 2250 + }, + { + "entropy": 0.39846932888031006, + "epoch": 5.913770913770914, + "grad_norm": 1.058344841003418, + "learning_rate": 0.00016892764973018475, + "loss": 0.3548299407958984, + "mean_token_accuracy": 0.8812332856655121, + "num_tokens": 3119763.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4225990093245949, + "eval_loss": 0.8478386402130127, + "eval_mean_token_accuracy": 0.7951162933688802, + "eval_num_tokens": 3162876.0, + "eval_runtime": 76.4757, + "eval_samples_per_second": 20.216, + "eval_steps_per_second": 2.537, + "step": 2334 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.174178661222349e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9719c9856db8e1cc1178fbbedbbeb71d9052cd46 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.02616102090367193, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..60c77ec31ce026439f9df029df62b416912f2b08 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2723/trainer_state.json @@ -0,0 +1,651 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2723, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.035328435897827, + "epoch": 0.1287001287001287, + "grad_norm": 3.0673413276672363, + "learning_rate": 4.96378417177596e-05, + "loss": 1.9008338928222657, + "mean_token_accuracy": 0.605284059047699, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.0828855693340302, + "epoch": 0.2574002574002574, + "grad_norm": 2.4980359077453613, + "learning_rate": 0.00010028870061343267, + "loss": 0.9762400817871094, + "mean_token_accuracy": 0.7393774157762527, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 0.9727641302347183, + "epoch": 0.3861003861003861, + "grad_norm": 1.5354057550430298, + "learning_rate": 0.00015093955950910574, + "loss": 0.8738698577880859, + "mean_token_accuracy": 0.761322969198227, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 0.9465948975086212, + "epoch": 0.5148005148005148, + "grad_norm": 1.3178573846817017, + "learning_rate": 0.00020159041840477876, + "loss": 0.854097671508789, + "mean_token_accuracy": 0.7649031269550324, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9218364179134368, + "epoch": 0.6435006435006435, + "grad_norm": 1.3482104539871216, + "learning_rate": 0.00025224127730045186, + "loss": 0.8339097595214844, + "mean_token_accuracy": 0.7713709837198257, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.92090303003788, + "epoch": 0.7722007722007722, + "grad_norm": 1.1728777885437012, + "learning_rate": 0.00030289213619612493, + "loss": 0.8308000183105468, + "mean_token_accuracy": 0.7717143112421035, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9001683223247529, + "epoch": 0.9009009009009009, + "grad_norm": 1.4331250190734863, + "learning_rate": 0.00035354299509179795, + "loss": 0.8269673919677735, + "mean_token_accuracy": 0.7711574441194534, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7020757987941664, + "eval_loss": 0.8044202923774719, + "eval_mean_token_accuracy": 0.7759947469553996, + "eval_num_tokens": 527146.0, + "eval_runtime": 78.3605, + "eval_samples_per_second": 19.729, + "eval_steps_per_second": 2.476, + "step": 389 + }, + { + "entropy": 0.8767053924425684, + "epoch": 1.0283140283140284, + "grad_norm": 1.3621660470962524, + "learning_rate": 0.0003940557495448841, + "loss": 0.8080394744873047, + "mean_token_accuracy": 0.7781236532962683, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8415434396266938, + "epoch": 1.157014157014157, + "grad_norm": 1.734777808189392, + "learning_rate": 0.00039377817338715463, + "loss": 0.7779766082763672, + "mean_token_accuracy": 0.7830026543140411, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8639528834819794, + "epoch": 1.2857142857142856, + "grad_norm": 1.3795595169067383, + "learning_rate": 0.00039310460257672286, + "loss": 0.7801459503173828, + "mean_token_accuracy": 0.7819079357385635, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.8538346707820892, + "epoch": 1.4144144144144144, + "grad_norm": 1.6661036014556885, + "learning_rate": 0.00039203639282089, + "loss": 0.7902972412109375, + "mean_token_accuracy": 0.7807946759462356, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8081058472394943, + "epoch": 1.5431145431145432, + "grad_norm": 1.2289175987243652, + "learning_rate": 0.00039057569412340117, + "loss": 0.7647433471679688, + "mean_token_accuracy": 0.7846344596147538, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8334913140535355, + "epoch": 1.6718146718146718, + "grad_norm": 2.2659049034118652, + "learning_rate": 0.0003887254464570972, + "loss": 0.7828218078613282, + "mean_token_accuracy": 0.7819693273305893, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8104432183504104, + "epoch": 1.8005148005148004, + "grad_norm": 1.7065047025680542, + "learning_rate": 0.00038648937384658125, + "loss": 0.763125, + "mean_token_accuracy": 0.7861280244588852, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8062294274568558, + "epoch": 1.9292149292149292, + "grad_norm": 1.3628491163253784, + "learning_rate": 0.0003838719768728124, + "loss": 0.7553546905517579, + "mean_token_accuracy": 0.7892439645528794, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.679631345665332, + "eval_loss": 0.7782689929008484, + "eval_mean_token_accuracy": 0.787303302398662, + "eval_num_tokens": 1054292.0, + "eval_runtime": 75.9942, + "eval_samples_per_second": 20.344, + "eval_steps_per_second": 2.553, + "step": 778 + }, + { + "entropy": 0.749360041786926, + "epoch": 2.056628056628057, + "grad_norm": 1.084079623222351, + "learning_rate": 0.0003808785236147101, + "loss": 0.7160065460205078, + "mean_token_accuracy": 0.7958323576233604, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.6975917857885361, + "epoch": 2.1853281853281854, + "grad_norm": 1.3939169645309448, + "learning_rate": 0.00037751503904600377, + "loss": 0.6710267639160157, + "mean_token_accuracy": 0.8034947836399078, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7072162652015686, + "epoch": 2.314028314028314, + "grad_norm": 1.9786381721496582, + "learning_rate": 0.0003737882929086661, + "loss": 0.6777350616455078, + "mean_token_accuracy": 0.8026440101861954, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7069408267736434, + "epoch": 2.4427284427284426, + "grad_norm": 1.3146620988845825, + "learning_rate": 0.00036970578608734, + "loss": 0.6795073699951172, + "mean_token_accuracy": 0.8013493180274963, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.6993900266289711, + "epoch": 2.571428571428571, + "grad_norm": 1.5064544677734375, + "learning_rate": 0.00036527573551218173, + "loss": 0.6791993713378907, + "mean_token_accuracy": 0.8022604930400848, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.6983578675985336, + "epoch": 2.7001287001287, + "grad_norm": 1.2354060411453247, + "learning_rate": 0.0003605070576205073, + "loss": 0.6693215179443359, + "mean_token_accuracy": 0.8064261078834534, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7105095881223679, + "epoch": 2.828828828828829, + "grad_norm": 1.182181477546692, + "learning_rate": 0.00035540935041052915, + "loss": 0.6762094116210937, + "mean_token_accuracy": 0.8052380502223968, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7148650509119033, + "epoch": 2.9575289575289574, + "grad_norm": 0.9586004018783569, + "learning_rate": 0.00034999287412330383, + "loss": 0.675987777709961, + "mean_token_accuracy": 0.8049957078695297, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6378419605112567, + "eval_loss": 0.7481877207756042, + "eval_mean_token_accuracy": 0.7935938085477376, + "eval_num_tokens": 1581438.0, + "eval_runtime": 75.8368, + "eval_samples_per_second": 20.386, + "eval_steps_per_second": 2.558, + "step": 1167 + }, + { + "entropy": 0.6485739145616088, + "epoch": 3.0849420849420848, + "grad_norm": 1.1198654174804688, + "learning_rate": 0.00034426853059177186, + "loss": 0.6087686157226563, + "mean_token_accuracy": 0.8164294560750326, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6030477187037468, + "epoch": 3.213642213642214, + "grad_norm": 1.1516684293746948, + "learning_rate": 0.00033824784129845556, + "loss": 0.5574645614624023, + "mean_token_accuracy": 0.8271971315145492, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6140900093317032, + "epoch": 3.3423423423423424, + "grad_norm": 1.372568964958191, + "learning_rate": 0.00033194292418597715, + "loss": 0.5769831848144531, + "mean_token_accuracy": 0.8246584522724152, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6223019778728485, + "epoch": 3.471042471042471, + "grad_norm": 1.5307430028915405, + "learning_rate": 0.0003253664692670717, + "loss": 0.5899515914916992, + "mean_token_accuracy": 0.820688226222992, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6215043878555297, + "epoch": 3.5997425997425996, + "grad_norm": 1.487574577331543, + "learning_rate": 0.0003185317130831854, + "loss": 0.5944060134887695, + "mean_token_accuracy": 0.8215733963251114, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6398005878925324, + "epoch": 3.7284427284427286, + "grad_norm": 1.054810881614685, + "learning_rate": 0.0003114524120630655, + "loss": 0.5942705154418946, + "mean_token_accuracy": 0.8215270626544953, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6384156176447868, + "epoch": 3.857142857142857, + "grad_norm": 1.5161454677581787, + "learning_rate": 0.00030414281483496505, + "loss": 0.5947936630249023, + "mean_token_accuracy": 0.8206489557027816, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.598966498374939, + "epoch": 3.985842985842986, + "grad_norm": 1.2136448621749878, + "learning_rate": 0.00029661763354818884, + "loss": 0.5746045684814454, + "mean_token_accuracy": 0.8274296247959136, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5822035986738107, + "eval_loss": 0.7569039463996887, + "eval_mean_token_accuracy": 0.7907421751735136, + "eval_num_tokens": 2108584.0, + "eval_runtime": 76.165, + "eval_samples_per_second": 20.298, + "eval_steps_per_second": 2.547, + "step": 1556 + }, + { + "entropy": 0.5126810152121265, + "epoch": 4.113256113256114, + "grad_norm": 1.2219198942184448, + "learning_rate": 0.00028889201426170327, + "loss": 0.4712484359741211, + "mean_token_accuracy": 0.8507941756585632, + "num_tokens": 2165782.0, + "step": 1600 + }, + { + "entropy": 0.5017523977160454, + "epoch": 4.241956241956242, + "grad_norm": 1.1246048212051392, + "learning_rate": 0.00028098150645940856, + "loss": 0.45758853912353514, + "mean_token_accuracy": 0.8549614226818085, + "num_tokens": 2233206.0, + "step": 1650 + }, + { + "entropy": 0.5128893256187439, + "epoch": 4.370656370656371, + "grad_norm": 1.2206052541732788, + "learning_rate": 0.000272902031753431, + "loss": 0.4767184066772461, + "mean_token_accuracy": 0.8486203062534332, + "num_tokens": 2301399.0, + "step": 1700 + }, + { + "entropy": 0.500448043346405, + "epoch": 4.499356499356499, + "grad_norm": 1.099212884902954, + "learning_rate": 0.0002646698518384262, + "loss": 0.46724525451660154, + "mean_token_accuracy": 0.8502935725450516, + "num_tokens": 2370831.0, + "step": 1750 + }, + { + "entropy": 0.5091731676459312, + "epoch": 4.628056628056628, + "grad_norm": 1.0495911836624146, + "learning_rate": 0.0002563015357613929, + "loss": 0.4759058380126953, + "mean_token_accuracy": 0.8488407224416733, + "num_tokens": 2439168.0, + "step": 1800 + }, + { + "entropy": 0.515411552786827, + "epoch": 4.756756756756757, + "grad_norm": 0.9209436774253845, + "learning_rate": 0.00024781392657287327, + "loss": 0.48281356811523435, + "mean_token_accuracy": 0.8456312155723572, + "num_tokens": 2508036.0, + "step": 1850 + }, + { + "entropy": 0.5016480022668839, + "epoch": 4.885456885456885, + "grad_norm": 1.5305742025375366, + "learning_rate": 0.00023922410742666182, + "loss": 0.4707856369018555, + "mean_token_accuracy": 0.8486748856306076, + "num_tokens": 2577025.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5264220541899967, + "eval_loss": 0.7557247281074524, + "eval_mean_token_accuracy": 0.7990376009154565, + "eval_num_tokens": 2635730.0, + "eval_runtime": 76.2827, + "eval_samples_per_second": 20.267, + "eval_steps_per_second": 2.543, + "step": 1945 + }, + { + "entropy": 0.5138596534126937, + "epoch": 5.012870012870013, + "grad_norm": 0.7728239893913269, + "learning_rate": 0.00023054936719625412, + "loss": 0.4731882095336914, + "mean_token_accuracy": 0.8493862681918674, + "num_tokens": 2642803.0, + "step": 1950 + }, + { + "entropy": 0.3956746557354927, + "epoch": 5.141570141570142, + "grad_norm": 0.852727472782135, + "learning_rate": 0.00022180716567724064, + "loss": 0.3441731262207031, + "mean_token_accuracy": 0.8864887124300003, + "num_tokens": 2708703.0, + "step": 2000 + }, + { + "entropy": 0.3931361359357834, + "epoch": 5.27027027027027, + "grad_norm": 1.062345266342163, + "learning_rate": 0.00021301509844568218, + "loss": 0.34529010772705077, + "mean_token_accuracy": 0.88379523396492, + "num_tokens": 2777956.0, + "step": 2050 + }, + { + "entropy": 0.3996828556060791, + "epoch": 5.398970398970399, + "grad_norm": 1.1475750207901, + "learning_rate": 0.00020419086144319811, + "loss": 0.35055686950683596, + "mean_token_accuracy": 0.8822557419538498, + "num_tokens": 2847032.0, + "step": 2100 + }, + { + "entropy": 0.3957197970151901, + "epoch": 5.527670527670527, + "grad_norm": 0.9748324751853943, + "learning_rate": 0.00019535221536004678, + "loss": 0.3523390197753906, + "mean_token_accuracy": 0.8818045473098755, + "num_tokens": 2915746.0, + "step": 2150 + }, + { + "entropy": 0.39835822969675067, + "epoch": 5.656370656370656, + "grad_norm": 1.1422828435897827, + "learning_rate": 0.00018651694988788578, + "loss": 0.35444053649902346, + "mean_token_accuracy": 0.8799562251567841, + "num_tokens": 2984346.0, + "step": 2200 + }, + { + "entropy": 0.40228330612182617, + "epoch": 5.785070785070785, + "grad_norm": 0.9757609963417053, + "learning_rate": 0.0001777028479141599, + "loss": 0.3603257751464844, + "mean_token_accuracy": 0.8790501236915589, + "num_tokens": 3051863.0, + "step": 2250 + }, + { + "entropy": 0.39846932888031006, + "epoch": 5.913770913770914, + "grad_norm": 1.058344841003418, + "learning_rate": 0.00016892764973018475, + "loss": 0.3548299407958984, + "mean_token_accuracy": 0.8812332856655121, + "num_tokens": 3119763.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4225990093245949, + "eval_loss": 0.8478386402130127, + "eval_mean_token_accuracy": 0.7951162933688802, + "eval_num_tokens": 3162876.0, + "eval_runtime": 76.4757, + "eval_samples_per_second": 20.216, + "eval_steps_per_second": 2.537, + "step": 2334 + }, + { + "entropy": 0.35978798884333985, + "epoch": 6.041184041184041, + "grad_norm": 1.1438041925430298, + "learning_rate": 0.00016020901732496375, + "loss": 0.3136854934692383, + "mean_token_accuracy": 0.8952204982439677, + "num_tokens": 3185495.0, + "step": 2350 + }, + { + "entropy": 0.274916353225708, + "epoch": 6.1698841698841695, + "grad_norm": 1.063818097114563, + "learning_rate": 0.000151564498836606, + "loss": 0.22996776580810546, + "mean_token_accuracy": 0.9221211689710617, + "num_tokens": 3252659.0, + "step": 2400 + }, + { + "entropy": 0.27972787141799926, + "epoch": 6.298584298584299, + "grad_norm": 0.9946160316467285, + "learning_rate": 0.00014301149323289332, + "loss": 0.23536941528320313, + "mean_token_accuracy": 0.9205723887681961, + "num_tokens": 3320155.0, + "step": 2450 + }, + { + "entropy": 0.27985903441905974, + "epoch": 6.427284427284428, + "grad_norm": 1.090388536453247, + "learning_rate": 0.00013456721529208496, + "loss": 0.23937501907348632, + "mean_token_accuracy": 0.9185256743431092, + "num_tokens": 3384147.0, + "step": 2500 + }, + { + "entropy": 0.27657748594880105, + "epoch": 6.555984555984556, + "grad_norm": 0.9090324640274048, + "learning_rate": 0.00012624866095444425, + "loss": 0.23602998733520508, + "mean_token_accuracy": 0.9194209152460098, + "num_tokens": 3453826.0, + "step": 2550 + }, + { + "entropy": 0.28343546360731126, + "epoch": 6.684684684684685, + "grad_norm": 0.956750750541687, + "learning_rate": 0.00011807257311422335, + "loss": 0.24055482864379882, + "mean_token_accuracy": 0.9179503041505813, + "num_tokens": 3521099.0, + "step": 2600 + }, + { + "entropy": 0.2764540535211563, + "epoch": 6.813384813384813, + "grad_norm": 0.9264359474182129, + "learning_rate": 0.0001100554079209585, + "loss": 0.23727087020874024, + "mean_token_accuracy": 0.9196303296089172, + "num_tokens": 3588515.0, + "step": 2650 + }, + { + "entropy": 0.27410337537527085, + "epoch": 6.942084942084942, + "grad_norm": 0.9057352542877197, + "learning_rate": 0.00010221330165790096, + "loss": 0.23279956817626954, + "mean_token_accuracy": 0.9197283333539963, + "num_tokens": 3658319.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.3525028199571924, + "eval_loss": 0.9255094528198242, + "eval_mean_token_accuracy": 0.7976236954792258, + "eval_num_tokens": 3690022.0, + "eval_runtime": 76.074, + "eval_samples_per_second": 20.322, + "eval_steps_per_second": 2.55, + "step": 2723 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.035462367328166e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9719c9856db8e1cc1178fbbedbbeb71d9052cd46 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.02616102090367193, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7162e19253a1cc07ee98b1020c2a5655dee76ed1 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3112/trainer_state.json @@ -0,0 +1,742 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3112, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.035328435897827, + "epoch": 0.1287001287001287, + "grad_norm": 3.0673413276672363, + "learning_rate": 4.96378417177596e-05, + "loss": 1.9008338928222657, + "mean_token_accuracy": 0.605284059047699, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.0828855693340302, + "epoch": 0.2574002574002574, + "grad_norm": 2.4980359077453613, + "learning_rate": 0.00010028870061343267, + "loss": 0.9762400817871094, + "mean_token_accuracy": 0.7393774157762527, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 0.9727641302347183, + "epoch": 0.3861003861003861, + "grad_norm": 1.5354057550430298, + "learning_rate": 0.00015093955950910574, + "loss": 0.8738698577880859, + "mean_token_accuracy": 0.761322969198227, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 0.9465948975086212, + "epoch": 0.5148005148005148, + "grad_norm": 1.3178573846817017, + "learning_rate": 0.00020159041840477876, + "loss": 0.854097671508789, + "mean_token_accuracy": 0.7649031269550324, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9218364179134368, + "epoch": 0.6435006435006435, + "grad_norm": 1.3482104539871216, + "learning_rate": 0.00025224127730045186, + "loss": 0.8339097595214844, + "mean_token_accuracy": 0.7713709837198257, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.92090303003788, + "epoch": 0.7722007722007722, + "grad_norm": 1.1728777885437012, + "learning_rate": 0.00030289213619612493, + "loss": 0.8308000183105468, + "mean_token_accuracy": 0.7717143112421035, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9001683223247529, + "epoch": 0.9009009009009009, + "grad_norm": 1.4331250190734863, + "learning_rate": 0.00035354299509179795, + "loss": 0.8269673919677735, + "mean_token_accuracy": 0.7711574441194534, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7020757987941664, + "eval_loss": 0.8044202923774719, + "eval_mean_token_accuracy": 0.7759947469553996, + "eval_num_tokens": 527146.0, + "eval_runtime": 78.3605, + "eval_samples_per_second": 19.729, + "eval_steps_per_second": 2.476, + "step": 389 + }, + { + "entropy": 0.8767053924425684, + "epoch": 1.0283140283140284, + "grad_norm": 1.3621660470962524, + "learning_rate": 0.0003940557495448841, + "loss": 0.8080394744873047, + "mean_token_accuracy": 0.7781236532962683, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8415434396266938, + "epoch": 1.157014157014157, + "grad_norm": 1.734777808189392, + "learning_rate": 0.00039377817338715463, + "loss": 0.7779766082763672, + "mean_token_accuracy": 0.7830026543140411, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8639528834819794, + "epoch": 1.2857142857142856, + "grad_norm": 1.3795595169067383, + "learning_rate": 0.00039310460257672286, + "loss": 0.7801459503173828, + "mean_token_accuracy": 0.7819079357385635, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.8538346707820892, + "epoch": 1.4144144144144144, + "grad_norm": 1.6661036014556885, + "learning_rate": 0.00039203639282089, + "loss": 0.7902972412109375, + "mean_token_accuracy": 0.7807946759462356, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8081058472394943, + "epoch": 1.5431145431145432, + "grad_norm": 1.2289175987243652, + "learning_rate": 0.00039057569412340117, + "loss": 0.7647433471679688, + "mean_token_accuracy": 0.7846344596147538, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8334913140535355, + "epoch": 1.6718146718146718, + "grad_norm": 2.2659049034118652, + "learning_rate": 0.0003887254464570972, + "loss": 0.7828218078613282, + "mean_token_accuracy": 0.7819693273305893, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8104432183504104, + "epoch": 1.8005148005148004, + "grad_norm": 1.7065047025680542, + "learning_rate": 0.00038648937384658125, + "loss": 0.763125, + "mean_token_accuracy": 0.7861280244588852, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8062294274568558, + "epoch": 1.9292149292149292, + "grad_norm": 1.3628491163253784, + "learning_rate": 0.0003838719768728124, + "loss": 0.7553546905517579, + "mean_token_accuracy": 0.7892439645528794, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.679631345665332, + "eval_loss": 0.7782689929008484, + "eval_mean_token_accuracy": 0.787303302398662, + "eval_num_tokens": 1054292.0, + "eval_runtime": 75.9942, + "eval_samples_per_second": 20.344, + "eval_steps_per_second": 2.553, + "step": 778 + }, + { + "entropy": 0.749360041786926, + "epoch": 2.056628056628057, + "grad_norm": 1.084079623222351, + "learning_rate": 0.0003808785236147101, + "loss": 0.7160065460205078, + "mean_token_accuracy": 0.7958323576233604, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.6975917857885361, + "epoch": 2.1853281853281854, + "grad_norm": 1.3939169645309448, + "learning_rate": 0.00037751503904600377, + "loss": 0.6710267639160157, + "mean_token_accuracy": 0.8034947836399078, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7072162652015686, + "epoch": 2.314028314028314, + "grad_norm": 1.9786381721496582, + "learning_rate": 0.0003737882929086661, + "loss": 0.6777350616455078, + "mean_token_accuracy": 0.8026440101861954, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7069408267736434, + "epoch": 2.4427284427284426, + "grad_norm": 1.3146620988845825, + "learning_rate": 0.00036970578608734, + "loss": 0.6795073699951172, + "mean_token_accuracy": 0.8013493180274963, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.6993900266289711, + "epoch": 2.571428571428571, + "grad_norm": 1.5064544677734375, + "learning_rate": 0.00036527573551218173, + "loss": 0.6791993713378907, + "mean_token_accuracy": 0.8022604930400848, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.6983578675985336, + "epoch": 2.7001287001287, + "grad_norm": 1.2354060411453247, + "learning_rate": 0.0003605070576205073, + "loss": 0.6693215179443359, + "mean_token_accuracy": 0.8064261078834534, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7105095881223679, + "epoch": 2.828828828828829, + "grad_norm": 1.182181477546692, + "learning_rate": 0.00035540935041052915, + "loss": 0.6762094116210937, + "mean_token_accuracy": 0.8052380502223968, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7148650509119033, + "epoch": 2.9575289575289574, + "grad_norm": 0.9586004018783569, + "learning_rate": 0.00034999287412330383, + "loss": 0.675987777709961, + "mean_token_accuracy": 0.8049957078695297, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6378419605112567, + "eval_loss": 0.7481877207756042, + "eval_mean_token_accuracy": 0.7935938085477376, + "eval_num_tokens": 1581438.0, + "eval_runtime": 75.8368, + "eval_samples_per_second": 20.386, + "eval_steps_per_second": 2.558, + "step": 1167 + }, + { + "entropy": 0.6485739145616088, + "epoch": 3.0849420849420848, + "grad_norm": 1.1198654174804688, + "learning_rate": 0.00034426853059177186, + "loss": 0.6087686157226563, + "mean_token_accuracy": 0.8164294560750326, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6030477187037468, + "epoch": 3.213642213642214, + "grad_norm": 1.1516684293746948, + "learning_rate": 0.00033824784129845556, + "loss": 0.5574645614624023, + "mean_token_accuracy": 0.8271971315145492, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6140900093317032, + "epoch": 3.3423423423423424, + "grad_norm": 1.372568964958191, + "learning_rate": 0.00033194292418597715, + "loss": 0.5769831848144531, + "mean_token_accuracy": 0.8246584522724152, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6223019778728485, + "epoch": 3.471042471042471, + "grad_norm": 1.5307430028915405, + "learning_rate": 0.0003253664692670717, + "loss": 0.5899515914916992, + "mean_token_accuracy": 0.820688226222992, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6215043878555297, + "epoch": 3.5997425997425996, + "grad_norm": 1.487574577331543, + "learning_rate": 0.0003185317130831854, + "loss": 0.5944060134887695, + "mean_token_accuracy": 0.8215733963251114, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6398005878925324, + "epoch": 3.7284427284427286, + "grad_norm": 1.054810881614685, + "learning_rate": 0.0003114524120630655, + "loss": 0.5942705154418946, + "mean_token_accuracy": 0.8215270626544953, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6384156176447868, + "epoch": 3.857142857142857, + "grad_norm": 1.5161454677581787, + "learning_rate": 0.00030414281483496505, + "loss": 0.5947936630249023, + "mean_token_accuracy": 0.8206489557027816, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.598966498374939, + "epoch": 3.985842985842986, + "grad_norm": 1.2136448621749878, + "learning_rate": 0.00029661763354818884, + "loss": 0.5746045684814454, + "mean_token_accuracy": 0.8274296247959136, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5822035986738107, + "eval_loss": 0.7569039463996887, + "eval_mean_token_accuracy": 0.7907421751735136, + "eval_num_tokens": 2108584.0, + "eval_runtime": 76.165, + "eval_samples_per_second": 20.298, + "eval_steps_per_second": 2.547, + "step": 1556 + }, + { + "entropy": 0.5126810152121265, + "epoch": 4.113256113256114, + "grad_norm": 1.2219198942184448, + "learning_rate": 0.00028889201426170327, + "loss": 0.4712484359741211, + "mean_token_accuracy": 0.8507941756585632, + "num_tokens": 2165782.0, + "step": 1600 + }, + { + "entropy": 0.5017523977160454, + "epoch": 4.241956241956242, + "grad_norm": 1.1246048212051392, + "learning_rate": 0.00028098150645940856, + "loss": 0.45758853912353514, + "mean_token_accuracy": 0.8549614226818085, + "num_tokens": 2233206.0, + "step": 1650 + }, + { + "entropy": 0.5128893256187439, + "epoch": 4.370656370656371, + "grad_norm": 1.2206052541732788, + "learning_rate": 0.000272902031753431, + "loss": 0.4767184066772461, + "mean_token_accuracy": 0.8486203062534332, + "num_tokens": 2301399.0, + "step": 1700 + }, + { + "entropy": 0.500448043346405, + "epoch": 4.499356499356499, + "grad_norm": 1.099212884902954, + "learning_rate": 0.0002646698518384262, + "loss": 0.46724525451660154, + "mean_token_accuracy": 0.8502935725450516, + "num_tokens": 2370831.0, + "step": 1750 + }, + { + "entropy": 0.5091731676459312, + "epoch": 4.628056628056628, + "grad_norm": 1.0495911836624146, + "learning_rate": 0.0002563015357613929, + "loss": 0.4759058380126953, + "mean_token_accuracy": 0.8488407224416733, + "num_tokens": 2439168.0, + "step": 1800 + }, + { + "entropy": 0.515411552786827, + "epoch": 4.756756756756757, + "grad_norm": 0.9209436774253845, + "learning_rate": 0.00024781392657287327, + "loss": 0.48281356811523435, + "mean_token_accuracy": 0.8456312155723572, + "num_tokens": 2508036.0, + "step": 1850 + }, + { + "entropy": 0.5016480022668839, + "epoch": 4.885456885456885, + "grad_norm": 1.5305742025375366, + "learning_rate": 0.00023922410742666182, + "loss": 0.4707856369018555, + "mean_token_accuracy": 0.8486748856306076, + "num_tokens": 2577025.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5264220541899967, + "eval_loss": 0.7557247281074524, + "eval_mean_token_accuracy": 0.7990376009154565, + "eval_num_tokens": 2635730.0, + "eval_runtime": 76.2827, + "eval_samples_per_second": 20.267, + "eval_steps_per_second": 2.543, + "step": 1945 + }, + { + "entropy": 0.5138596534126937, + "epoch": 5.012870012870013, + "grad_norm": 0.7728239893913269, + "learning_rate": 0.00023054936719625412, + "loss": 0.4731882095336914, + "mean_token_accuracy": 0.8493862681918674, + "num_tokens": 2642803.0, + "step": 1950 + }, + { + "entropy": 0.3956746557354927, + "epoch": 5.141570141570142, + "grad_norm": 0.852727472782135, + "learning_rate": 0.00022180716567724064, + "loss": 0.3441731262207031, + "mean_token_accuracy": 0.8864887124300003, + "num_tokens": 2708703.0, + "step": 2000 + }, + { + "entropy": 0.3931361359357834, + "epoch": 5.27027027027027, + "grad_norm": 1.062345266342163, + "learning_rate": 0.00021301509844568218, + "loss": 0.34529010772705077, + "mean_token_accuracy": 0.88379523396492, + "num_tokens": 2777956.0, + "step": 2050 + }, + { + "entropy": 0.3996828556060791, + "epoch": 5.398970398970399, + "grad_norm": 1.1475750207901, + "learning_rate": 0.00020419086144319811, + "loss": 0.35055686950683596, + "mean_token_accuracy": 0.8822557419538498, + "num_tokens": 2847032.0, + "step": 2100 + }, + { + "entropy": 0.3957197970151901, + "epoch": 5.527670527670527, + "grad_norm": 0.9748324751853943, + "learning_rate": 0.00019535221536004678, + "loss": 0.3523390197753906, + "mean_token_accuracy": 0.8818045473098755, + "num_tokens": 2915746.0, + "step": 2150 + }, + { + "entropy": 0.39835822969675067, + "epoch": 5.656370656370656, + "grad_norm": 1.1422828435897827, + "learning_rate": 0.00018651694988788578, + "loss": 0.35444053649902346, + "mean_token_accuracy": 0.8799562251567841, + "num_tokens": 2984346.0, + "step": 2200 + }, + { + "entropy": 0.40228330612182617, + "epoch": 5.785070785070785, + "grad_norm": 0.9757609963417053, + "learning_rate": 0.0001777028479141599, + "loss": 0.3603257751464844, + "mean_token_accuracy": 0.8790501236915589, + "num_tokens": 3051863.0, + "step": 2250 + }, + { + "entropy": 0.39846932888031006, + "epoch": 5.913770913770914, + "grad_norm": 1.058344841003418, + "learning_rate": 0.00016892764973018475, + "loss": 0.3548299407958984, + "mean_token_accuracy": 0.8812332856655121, + "num_tokens": 3119763.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4225990093245949, + "eval_loss": 0.8478386402130127, + "eval_mean_token_accuracy": 0.7951162933688802, + "eval_num_tokens": 3162876.0, + "eval_runtime": 76.4757, + "eval_samples_per_second": 20.216, + "eval_steps_per_second": 2.537, + "step": 2334 + }, + { + "entropy": 0.35978798884333985, + "epoch": 6.041184041184041, + "grad_norm": 1.1438041925430298, + "learning_rate": 0.00016020901732496375, + "loss": 0.3136854934692383, + "mean_token_accuracy": 0.8952204982439677, + "num_tokens": 3185495.0, + "step": 2350 + }, + { + "entropy": 0.274916353225708, + "epoch": 6.1698841698841695, + "grad_norm": 1.063818097114563, + "learning_rate": 0.000151564498836606, + "loss": 0.22996776580810546, + "mean_token_accuracy": 0.9221211689710617, + "num_tokens": 3252659.0, + "step": 2400 + }, + { + "entropy": 0.27972787141799926, + "epoch": 6.298584298584299, + "grad_norm": 0.9946160316467285, + "learning_rate": 0.00014301149323289332, + "loss": 0.23536941528320313, + "mean_token_accuracy": 0.9205723887681961, + "num_tokens": 3320155.0, + "step": 2450 + }, + { + "entropy": 0.27985903441905974, + "epoch": 6.427284427284428, + "grad_norm": 1.090388536453247, + "learning_rate": 0.00013456721529208496, + "loss": 0.23937501907348632, + "mean_token_accuracy": 0.9185256743431092, + "num_tokens": 3384147.0, + "step": 2500 + }, + { + "entropy": 0.27657748594880105, + "epoch": 6.555984555984556, + "grad_norm": 0.9090324640274048, + "learning_rate": 0.00012624866095444425, + "loss": 0.23602998733520508, + "mean_token_accuracy": 0.9194209152460098, + "num_tokens": 3453826.0, + "step": 2550 + }, + { + "entropy": 0.28343546360731126, + "epoch": 6.684684684684685, + "grad_norm": 0.956750750541687, + "learning_rate": 0.00011807257311422335, + "loss": 0.24055482864379882, + "mean_token_accuracy": 0.9179503041505813, + "num_tokens": 3521099.0, + "step": 2600 + }, + { + "entropy": 0.2764540535211563, + "epoch": 6.813384813384813, + "grad_norm": 0.9264359474182129, + "learning_rate": 0.0001100554079209585, + "loss": 0.23727087020874024, + "mean_token_accuracy": 0.9196303296089172, + "num_tokens": 3588515.0, + "step": 2650 + }, + { + "entropy": 0.27410337537527085, + "epoch": 6.942084942084942, + "grad_norm": 0.9057352542877197, + "learning_rate": 0.00010221330165790096, + "loss": 0.23279956817626954, + "mean_token_accuracy": 0.9197283333539963, + "num_tokens": 3658319.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.3525028199571924, + "eval_loss": 0.9255094528198242, + "eval_mean_token_accuracy": 0.7976236954792258, + "eval_num_tokens": 3690022.0, + "eval_runtime": 76.074, + "eval_samples_per_second": 20.322, + "eval_steps_per_second": 2.55, + "step": 2723 + }, + { + "entropy": 0.22481059982921137, + "epoch": 7.06949806949807, + "grad_norm": 1.1758164167404175, + "learning_rate": 9.456203826424829e-05, + "loss": 0.1848788833618164, + "mean_token_accuracy": 0.9379562127469766, + "num_tokens": 3727811.0, + "step": 2750 + }, + { + "entropy": 0.18282695084810258, + "epoch": 7.198198198198198, + "grad_norm": 0.781952977180481, + "learning_rate": 8.711701756654426e-05, + "loss": 0.1484829616546631, + "mean_token_accuracy": 0.9501194298267365, + "num_tokens": 3795920.0, + "step": 2800 + }, + { + "entropy": 0.18234790086746216, + "epoch": 7.326898326898327, + "grad_norm": 0.703240692615509, + "learning_rate": 7.989322428318929e-05, + "loss": 0.15111746788024902, + "mean_token_accuracy": 0.9502605730295182, + "num_tokens": 3863324.0, + "step": 2850 + }, + { + "entropy": 0.1816841357946396, + "epoch": 7.455598455598455, + "grad_norm": 1.0166149139404297, + "learning_rate": 7.290519786444519e-05, + "loss": 0.15027329444885254, + "mean_token_accuracy": 0.9499351298809051, + "num_tokens": 3932191.0, + "step": 2900 + }, + { + "entropy": 0.17630216613411903, + "epoch": 7.584298584298584, + "grad_norm": 0.5356483459472656, + "learning_rate": 6.616700322863927e-05, + "loss": 0.14939964294433594, + "mean_token_accuracy": 0.9505619239807129, + "num_tokens": 4000632.0, + "step": 2950 + }, + { + "entropy": 0.18283376961946488, + "epoch": 7.712998712998713, + "grad_norm": 0.7002793550491333, + "learning_rate": 5.969220245346672e-05, + "loss": 0.15329421997070314, + "mean_token_accuracy": 0.949188598394394, + "num_tokens": 4067753.0, + "step": 3000 + }, + { + "entropy": 0.18244779869914055, + "epoch": 7.841698841698841, + "grad_norm": 0.6270423531532288, + "learning_rate": 5.349382747936841e-05, + "loss": 0.15302456855773927, + "mean_token_accuracy": 0.9504549181461335, + "num_tokens": 4133681.0, + "step": 3050 + }, + { + "entropy": 0.17880768597126007, + "epoch": 7.97039897039897, + "grad_norm": 0.6172229647636414, + "learning_rate": 4.758435387992531e-05, + "loss": 0.14883087158203126, + "mean_token_accuracy": 0.9509924799203873, + "num_tokens": 4201263.0, + "step": 3100 + }, + { + "epoch": 8.0, + "eval_entropy": 0.28235290898490195, + "eval_loss": 1.1443017721176147, + "eval_mean_token_accuracy": 0.7958696088225571, + "eval_num_tokens": 4217168.0, + "eval_runtime": 76.1084, + "eval_samples_per_second": 20.313, + "eval_steps_per_second": 2.549, + "step": 3112 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.894902103240998e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9719c9856db8e1cc1178fbbedbbeb71d9052cd46 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.02616102090367193, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d5ea3885c82247378af14ffb0805856f8ab5d636 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3501/trainer_state.json @@ -0,0 +1,833 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3501, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.035328435897827, + "epoch": 0.1287001287001287, + "grad_norm": 3.0673413276672363, + "learning_rate": 4.96378417177596e-05, + "loss": 1.9008338928222657, + "mean_token_accuracy": 0.605284059047699, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.0828855693340302, + "epoch": 0.2574002574002574, + "grad_norm": 2.4980359077453613, + "learning_rate": 0.00010028870061343267, + "loss": 0.9762400817871094, + "mean_token_accuracy": 0.7393774157762527, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 0.9727641302347183, + "epoch": 0.3861003861003861, + "grad_norm": 1.5354057550430298, + "learning_rate": 0.00015093955950910574, + "loss": 0.8738698577880859, + "mean_token_accuracy": 0.761322969198227, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 0.9465948975086212, + "epoch": 0.5148005148005148, + "grad_norm": 1.3178573846817017, + "learning_rate": 0.00020159041840477876, + "loss": 0.854097671508789, + "mean_token_accuracy": 0.7649031269550324, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9218364179134368, + "epoch": 0.6435006435006435, + "grad_norm": 1.3482104539871216, + "learning_rate": 0.00025224127730045186, + "loss": 0.8339097595214844, + "mean_token_accuracy": 0.7713709837198257, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.92090303003788, + "epoch": 0.7722007722007722, + "grad_norm": 1.1728777885437012, + "learning_rate": 0.00030289213619612493, + "loss": 0.8308000183105468, + "mean_token_accuracy": 0.7717143112421035, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9001683223247529, + "epoch": 0.9009009009009009, + "grad_norm": 1.4331250190734863, + "learning_rate": 0.00035354299509179795, + "loss": 0.8269673919677735, + "mean_token_accuracy": 0.7711574441194534, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7020757987941664, + "eval_loss": 0.8044202923774719, + "eval_mean_token_accuracy": 0.7759947469553996, + "eval_num_tokens": 527146.0, + "eval_runtime": 78.3605, + "eval_samples_per_second": 19.729, + "eval_steps_per_second": 2.476, + "step": 389 + }, + { + "entropy": 0.8767053924425684, + "epoch": 1.0283140283140284, + "grad_norm": 1.3621660470962524, + "learning_rate": 0.0003940557495448841, + "loss": 0.8080394744873047, + "mean_token_accuracy": 0.7781236532962683, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8415434396266938, + "epoch": 1.157014157014157, + "grad_norm": 1.734777808189392, + "learning_rate": 0.00039377817338715463, + "loss": 0.7779766082763672, + "mean_token_accuracy": 0.7830026543140411, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8639528834819794, + "epoch": 1.2857142857142856, + "grad_norm": 1.3795595169067383, + "learning_rate": 0.00039310460257672286, + "loss": 0.7801459503173828, + "mean_token_accuracy": 0.7819079357385635, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.8538346707820892, + "epoch": 1.4144144144144144, + "grad_norm": 1.6661036014556885, + "learning_rate": 0.00039203639282089, + "loss": 0.7902972412109375, + "mean_token_accuracy": 0.7807946759462356, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8081058472394943, + "epoch": 1.5431145431145432, + "grad_norm": 1.2289175987243652, + "learning_rate": 0.00039057569412340117, + "loss": 0.7647433471679688, + "mean_token_accuracy": 0.7846344596147538, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8334913140535355, + "epoch": 1.6718146718146718, + "grad_norm": 2.2659049034118652, + "learning_rate": 0.0003887254464570972, + "loss": 0.7828218078613282, + "mean_token_accuracy": 0.7819693273305893, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8104432183504104, + "epoch": 1.8005148005148004, + "grad_norm": 1.7065047025680542, + "learning_rate": 0.00038648937384658125, + "loss": 0.763125, + "mean_token_accuracy": 0.7861280244588852, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8062294274568558, + "epoch": 1.9292149292149292, + "grad_norm": 1.3628491163253784, + "learning_rate": 0.0003838719768728124, + "loss": 0.7553546905517579, + "mean_token_accuracy": 0.7892439645528794, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.679631345665332, + "eval_loss": 0.7782689929008484, + "eval_mean_token_accuracy": 0.787303302398662, + "eval_num_tokens": 1054292.0, + "eval_runtime": 75.9942, + "eval_samples_per_second": 20.344, + "eval_steps_per_second": 2.553, + "step": 778 + }, + { + "entropy": 0.749360041786926, + "epoch": 2.056628056628057, + "grad_norm": 1.084079623222351, + "learning_rate": 0.0003808785236147101, + "loss": 0.7160065460205078, + "mean_token_accuracy": 0.7958323576233604, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.6975917857885361, + "epoch": 2.1853281853281854, + "grad_norm": 1.3939169645309448, + "learning_rate": 0.00037751503904600377, + "loss": 0.6710267639160157, + "mean_token_accuracy": 0.8034947836399078, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7072162652015686, + "epoch": 2.314028314028314, + "grad_norm": 1.9786381721496582, + "learning_rate": 0.0003737882929086661, + "loss": 0.6777350616455078, + "mean_token_accuracy": 0.8026440101861954, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7069408267736434, + "epoch": 2.4427284427284426, + "grad_norm": 1.3146620988845825, + "learning_rate": 0.00036970578608734, + "loss": 0.6795073699951172, + "mean_token_accuracy": 0.8013493180274963, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.6993900266289711, + "epoch": 2.571428571428571, + "grad_norm": 1.5064544677734375, + "learning_rate": 0.00036527573551218173, + "loss": 0.6791993713378907, + "mean_token_accuracy": 0.8022604930400848, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.6983578675985336, + "epoch": 2.7001287001287, + "grad_norm": 1.2354060411453247, + "learning_rate": 0.0003605070576205073, + "loss": 0.6693215179443359, + "mean_token_accuracy": 0.8064261078834534, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7105095881223679, + "epoch": 2.828828828828829, + "grad_norm": 1.182181477546692, + "learning_rate": 0.00035540935041052915, + "loss": 0.6762094116210937, + "mean_token_accuracy": 0.8052380502223968, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7148650509119033, + "epoch": 2.9575289575289574, + "grad_norm": 0.9586004018783569, + "learning_rate": 0.00034999287412330383, + "loss": 0.675987777709961, + "mean_token_accuracy": 0.8049957078695297, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6378419605112567, + "eval_loss": 0.7481877207756042, + "eval_mean_token_accuracy": 0.7935938085477376, + "eval_num_tokens": 1581438.0, + "eval_runtime": 75.8368, + "eval_samples_per_second": 20.386, + "eval_steps_per_second": 2.558, + "step": 1167 + }, + { + "entropy": 0.6485739145616088, + "epoch": 3.0849420849420848, + "grad_norm": 1.1198654174804688, + "learning_rate": 0.00034426853059177186, + "loss": 0.6087686157226563, + "mean_token_accuracy": 0.8164294560750326, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6030477187037468, + "epoch": 3.213642213642214, + "grad_norm": 1.1516684293746948, + "learning_rate": 0.00033824784129845556, + "loss": 0.5574645614624023, + "mean_token_accuracy": 0.8271971315145492, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6140900093317032, + "epoch": 3.3423423423423424, + "grad_norm": 1.372568964958191, + "learning_rate": 0.00033194292418597715, + "loss": 0.5769831848144531, + "mean_token_accuracy": 0.8246584522724152, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6223019778728485, + "epoch": 3.471042471042471, + "grad_norm": 1.5307430028915405, + "learning_rate": 0.0003253664692670717, + "loss": 0.5899515914916992, + "mean_token_accuracy": 0.820688226222992, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6215043878555297, + "epoch": 3.5997425997425996, + "grad_norm": 1.487574577331543, + "learning_rate": 0.0003185317130831854, + "loss": 0.5944060134887695, + "mean_token_accuracy": 0.8215733963251114, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6398005878925324, + "epoch": 3.7284427284427286, + "grad_norm": 1.054810881614685, + "learning_rate": 0.0003114524120630655, + "loss": 0.5942705154418946, + "mean_token_accuracy": 0.8215270626544953, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6384156176447868, + "epoch": 3.857142857142857, + "grad_norm": 1.5161454677581787, + "learning_rate": 0.00030414281483496505, + "loss": 0.5947936630249023, + "mean_token_accuracy": 0.8206489557027816, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.598966498374939, + "epoch": 3.985842985842986, + "grad_norm": 1.2136448621749878, + "learning_rate": 0.00029661763354818884, + "loss": 0.5746045684814454, + "mean_token_accuracy": 0.8274296247959136, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5822035986738107, + "eval_loss": 0.7569039463996887, + "eval_mean_token_accuracy": 0.7907421751735136, + "eval_num_tokens": 2108584.0, + "eval_runtime": 76.165, + "eval_samples_per_second": 20.298, + "eval_steps_per_second": 2.547, + "step": 1556 + }, + { + "entropy": 0.5126810152121265, + "epoch": 4.113256113256114, + "grad_norm": 1.2219198942184448, + "learning_rate": 0.00028889201426170327, + "loss": 0.4712484359741211, + "mean_token_accuracy": 0.8507941756585632, + "num_tokens": 2165782.0, + "step": 1600 + }, + { + "entropy": 0.5017523977160454, + "epoch": 4.241956241956242, + "grad_norm": 1.1246048212051392, + "learning_rate": 0.00028098150645940856, + "loss": 0.45758853912353514, + "mean_token_accuracy": 0.8549614226818085, + "num_tokens": 2233206.0, + "step": 1650 + }, + { + "entropy": 0.5128893256187439, + "epoch": 4.370656370656371, + "grad_norm": 1.2206052541732788, + "learning_rate": 0.000272902031753431, + "loss": 0.4767184066772461, + "mean_token_accuracy": 0.8486203062534332, + "num_tokens": 2301399.0, + "step": 1700 + }, + { + "entropy": 0.500448043346405, + "epoch": 4.499356499356499, + "grad_norm": 1.099212884902954, + "learning_rate": 0.0002646698518384262, + "loss": 0.46724525451660154, + "mean_token_accuracy": 0.8502935725450516, + "num_tokens": 2370831.0, + "step": 1750 + }, + { + "entropy": 0.5091731676459312, + "epoch": 4.628056628056628, + "grad_norm": 1.0495911836624146, + "learning_rate": 0.0002563015357613929, + "loss": 0.4759058380126953, + "mean_token_accuracy": 0.8488407224416733, + "num_tokens": 2439168.0, + "step": 1800 + }, + { + "entropy": 0.515411552786827, + "epoch": 4.756756756756757, + "grad_norm": 0.9209436774253845, + "learning_rate": 0.00024781392657287327, + "loss": 0.48281356811523435, + "mean_token_accuracy": 0.8456312155723572, + "num_tokens": 2508036.0, + "step": 1850 + }, + { + "entropy": 0.5016480022668839, + "epoch": 4.885456885456885, + "grad_norm": 1.5305742025375366, + "learning_rate": 0.00023922410742666182, + "loss": 0.4707856369018555, + "mean_token_accuracy": 0.8486748856306076, + "num_tokens": 2577025.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5264220541899967, + "eval_loss": 0.7557247281074524, + "eval_mean_token_accuracy": 0.7990376009154565, + "eval_num_tokens": 2635730.0, + "eval_runtime": 76.2827, + "eval_samples_per_second": 20.267, + "eval_steps_per_second": 2.543, + "step": 1945 + }, + { + "entropy": 0.5138596534126937, + "epoch": 5.012870012870013, + "grad_norm": 0.7728239893913269, + "learning_rate": 0.00023054936719625412, + "loss": 0.4731882095336914, + "mean_token_accuracy": 0.8493862681918674, + "num_tokens": 2642803.0, + "step": 1950 + }, + { + "entropy": 0.3956746557354927, + "epoch": 5.141570141570142, + "grad_norm": 0.852727472782135, + "learning_rate": 0.00022180716567724064, + "loss": 0.3441731262207031, + "mean_token_accuracy": 0.8864887124300003, + "num_tokens": 2708703.0, + "step": 2000 + }, + { + "entropy": 0.3931361359357834, + "epoch": 5.27027027027027, + "grad_norm": 1.062345266342163, + "learning_rate": 0.00021301509844568218, + "loss": 0.34529010772705077, + "mean_token_accuracy": 0.88379523396492, + "num_tokens": 2777956.0, + "step": 2050 + }, + { + "entropy": 0.3996828556060791, + "epoch": 5.398970398970399, + "grad_norm": 1.1475750207901, + "learning_rate": 0.00020419086144319811, + "loss": 0.35055686950683596, + "mean_token_accuracy": 0.8822557419538498, + "num_tokens": 2847032.0, + "step": 2100 + }, + { + "entropy": 0.3957197970151901, + "epoch": 5.527670527670527, + "grad_norm": 0.9748324751853943, + "learning_rate": 0.00019535221536004678, + "loss": 0.3523390197753906, + "mean_token_accuracy": 0.8818045473098755, + "num_tokens": 2915746.0, + "step": 2150 + }, + { + "entropy": 0.39835822969675067, + "epoch": 5.656370656370656, + "grad_norm": 1.1422828435897827, + "learning_rate": 0.00018651694988788578, + "loss": 0.35444053649902346, + "mean_token_accuracy": 0.8799562251567841, + "num_tokens": 2984346.0, + "step": 2200 + }, + { + "entropy": 0.40228330612182617, + "epoch": 5.785070785070785, + "grad_norm": 0.9757609963417053, + "learning_rate": 0.0001777028479141599, + "loss": 0.3603257751464844, + "mean_token_accuracy": 0.8790501236915589, + "num_tokens": 3051863.0, + "step": 2250 + }, + { + "entropy": 0.39846932888031006, + "epoch": 5.913770913770914, + "grad_norm": 1.058344841003418, + "learning_rate": 0.00016892764973018475, + "loss": 0.3548299407958984, + "mean_token_accuracy": 0.8812332856655121, + "num_tokens": 3119763.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4225990093245949, + "eval_loss": 0.8478386402130127, + "eval_mean_token_accuracy": 0.7951162933688802, + "eval_num_tokens": 3162876.0, + "eval_runtime": 76.4757, + "eval_samples_per_second": 20.216, + "eval_steps_per_second": 2.537, + "step": 2334 + }, + { + "entropy": 0.35978798884333985, + "epoch": 6.041184041184041, + "grad_norm": 1.1438041925430298, + "learning_rate": 0.00016020901732496375, + "loss": 0.3136854934692383, + "mean_token_accuracy": 0.8952204982439677, + "num_tokens": 3185495.0, + "step": 2350 + }, + { + "entropy": 0.274916353225708, + "epoch": 6.1698841698841695, + "grad_norm": 1.063818097114563, + "learning_rate": 0.000151564498836606, + "loss": 0.22996776580810546, + "mean_token_accuracy": 0.9221211689710617, + "num_tokens": 3252659.0, + "step": 2400 + }, + { + "entropy": 0.27972787141799926, + "epoch": 6.298584298584299, + "grad_norm": 0.9946160316467285, + "learning_rate": 0.00014301149323289332, + "loss": 0.23536941528320313, + "mean_token_accuracy": 0.9205723887681961, + "num_tokens": 3320155.0, + "step": 2450 + }, + { + "entropy": 0.27985903441905974, + "epoch": 6.427284427284428, + "grad_norm": 1.090388536453247, + "learning_rate": 0.00013456721529208496, + "loss": 0.23937501907348632, + "mean_token_accuracy": 0.9185256743431092, + "num_tokens": 3384147.0, + "step": 2500 + }, + { + "entropy": 0.27657748594880105, + "epoch": 6.555984555984556, + "grad_norm": 0.9090324640274048, + "learning_rate": 0.00012624866095444425, + "loss": 0.23602998733520508, + "mean_token_accuracy": 0.9194209152460098, + "num_tokens": 3453826.0, + "step": 2550 + }, + { + "entropy": 0.28343546360731126, + "epoch": 6.684684684684685, + "grad_norm": 0.956750750541687, + "learning_rate": 0.00011807257311422335, + "loss": 0.24055482864379882, + "mean_token_accuracy": 0.9179503041505813, + "num_tokens": 3521099.0, + "step": 2600 + }, + { + "entropy": 0.2764540535211563, + "epoch": 6.813384813384813, + "grad_norm": 0.9264359474182129, + "learning_rate": 0.0001100554079209585, + "loss": 0.23727087020874024, + "mean_token_accuracy": 0.9196303296089172, + "num_tokens": 3588515.0, + "step": 2650 + }, + { + "entropy": 0.27410337537527085, + "epoch": 6.942084942084942, + "grad_norm": 0.9057352542877197, + "learning_rate": 0.00010221330165790096, + "loss": 0.23279956817626954, + "mean_token_accuracy": 0.9197283333539963, + "num_tokens": 3658319.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.3525028199571924, + "eval_loss": 0.9255094528198242, + "eval_mean_token_accuracy": 0.7976236954792258, + "eval_num_tokens": 3690022.0, + "eval_runtime": 76.074, + "eval_samples_per_second": 20.322, + "eval_steps_per_second": 2.55, + "step": 2723 + }, + { + "entropy": 0.22481059982921137, + "epoch": 7.06949806949807, + "grad_norm": 1.1758164167404175, + "learning_rate": 9.456203826424829e-05, + "loss": 0.1848788833618164, + "mean_token_accuracy": 0.9379562127469766, + "num_tokens": 3727811.0, + "step": 2750 + }, + { + "entropy": 0.18282695084810258, + "epoch": 7.198198198198198, + "grad_norm": 0.781952977180481, + "learning_rate": 8.711701756654426e-05, + "loss": 0.1484829616546631, + "mean_token_accuracy": 0.9501194298267365, + "num_tokens": 3795920.0, + "step": 2800 + }, + { + "entropy": 0.18234790086746216, + "epoch": 7.326898326898327, + "grad_norm": 0.703240692615509, + "learning_rate": 7.989322428318929e-05, + "loss": 0.15111746788024902, + "mean_token_accuracy": 0.9502605730295182, + "num_tokens": 3863324.0, + "step": 2850 + }, + { + "entropy": 0.1816841357946396, + "epoch": 7.455598455598455, + "grad_norm": 1.0166149139404297, + "learning_rate": 7.290519786444519e-05, + "loss": 0.15027329444885254, + "mean_token_accuracy": 0.9499351298809051, + "num_tokens": 3932191.0, + "step": 2900 + }, + { + "entropy": 0.17630216613411903, + "epoch": 7.584298584298584, + "grad_norm": 0.5356483459472656, + "learning_rate": 6.616700322863927e-05, + "loss": 0.14939964294433594, + "mean_token_accuracy": 0.9505619239807129, + "num_tokens": 4000632.0, + "step": 2950 + }, + { + "entropy": 0.18283376961946488, + "epoch": 7.712998712998713, + "grad_norm": 0.7002793550491333, + "learning_rate": 5.969220245346672e-05, + "loss": 0.15329421997070314, + "mean_token_accuracy": 0.949188598394394, + "num_tokens": 4067753.0, + "step": 3000 + }, + { + "entropy": 0.18244779869914055, + "epoch": 7.841698841698841, + "grad_norm": 0.6270423531532288, + "learning_rate": 5.349382747936841e-05, + "loss": 0.15302456855773927, + "mean_token_accuracy": 0.9504549181461335, + "num_tokens": 4133681.0, + "step": 3050 + }, + { + "entropy": 0.17880768597126007, + "epoch": 7.97039897039897, + "grad_norm": 0.6172229647636414, + "learning_rate": 4.758435387992531e-05, + "loss": 0.14883087158203126, + "mean_token_accuracy": 0.9509924799203873, + "num_tokens": 4201263.0, + "step": 3100 + }, + { + "epoch": 8.0, + "eval_entropy": 0.28235290898490195, + "eval_loss": 1.1443017721176147, + "eval_mean_token_accuracy": 0.7958696088225571, + "eval_num_tokens": 4217168.0, + "eval_runtime": 76.1084, + "eval_samples_per_second": 20.313, + "eval_steps_per_second": 2.549, + "step": 3112 + }, + { + "entropy": 0.15051050589542198, + "epoch": 8.097812097812097, + "grad_norm": 0.4672251045703888, + "learning_rate": 4.197567575206164e-05, + "loss": 0.12156478881835937, + "mean_token_accuracy": 0.9611766500906511, + "num_tokens": 4268387.0, + "step": 3150 + }, + { + "entropy": 0.13742553241550923, + "epoch": 8.226512226512227, + "grad_norm": 0.7555282115936279, + "learning_rate": 3.667908177659604e-05, + "loss": 0.11531492233276368, + "mean_token_accuracy": 0.9618133813142776, + "num_tokens": 4335443.0, + "step": 3200 + }, + { + "entropy": 0.13783976405858994, + "epoch": 8.355212355212355, + "grad_norm": 0.372898131608963, + "learning_rate": 3.170523249732349e-05, + "loss": 0.11172324180603027, + "mean_token_accuracy": 0.9617116987705231, + "num_tokens": 4405738.0, + "step": 3250 + }, + { + "entropy": 0.1400664184987545, + "epoch": 8.483912483912484, + "grad_norm": 0.6359309554100037, + "learning_rate": 2.7064138864359793e-05, + "loss": 0.1197418212890625, + "mean_token_accuracy": 0.9595164811611175, + "num_tokens": 4471215.0, + "step": 3300 + }, + { + "entropy": 0.1393050680309534, + "epoch": 8.612612612612612, + "grad_norm": 0.47000861167907715, + "learning_rate": 2.2765142084933584e-05, + "loss": 0.11754611015319824, + "mean_token_accuracy": 0.9604016053676605, + "num_tokens": 4538643.0, + "step": 3350 + }, + { + "entropy": 0.14121561706066132, + "epoch": 8.741312741312742, + "grad_norm": 0.6012074947357178, + "learning_rate": 1.8816894822181725e-05, + "loss": 0.12010603904724121, + "mean_token_accuracy": 0.9601105463504791, + "num_tokens": 4604547.0, + "step": 3400 + }, + { + "entropy": 0.13598045460879804, + "epoch": 8.87001287001287, + "grad_norm": 0.44798487424850464, + "learning_rate": 1.522734377978824e-05, + "loss": 0.1126248836517334, + "mean_token_accuracy": 0.9618371516466141, + "num_tokens": 4675035.0, + "step": 3450 + }, + { + "entropy": 0.1365610294044018, + "epoch": 8.998712998712998, + "grad_norm": 0.3193477690219879, + "learning_rate": 1.2003713707520592e-05, + "loss": 0.11523623466491699, + "mean_token_accuracy": 0.961252732872963, + "num_tokens": 4743741.0, + "step": 3500 + }, + { + "epoch": 9.0, + "eval_entropy": 0.256610156641793, + "eval_loss": 1.2551246881484985, + "eval_mean_token_accuracy": 0.7978993546716946, + "eval_num_tokens": 4744314.0, + "eval_runtime": 77.606, + "eval_samples_per_second": 19.921, + "eval_steps_per_second": 2.5, + "step": 3501 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.758889278786048e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9719c9856db8e1cc1178fbbedbbeb71d9052cd46 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.02616102090367193, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e3c7ff3df3eb9256fa448cad91c0c5898e372417 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-389/trainer_state.json @@ -0,0 +1,115 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 389, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.035328435897827, + "epoch": 0.1287001287001287, + "grad_norm": 3.0673413276672363, + "learning_rate": 4.96378417177596e-05, + "loss": 1.9008338928222657, + "mean_token_accuracy": 0.605284059047699, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.0828855693340302, + "epoch": 0.2574002574002574, + "grad_norm": 2.4980359077453613, + "learning_rate": 0.00010028870061343267, + "loss": 0.9762400817871094, + "mean_token_accuracy": 0.7393774157762527, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 0.9727641302347183, + "epoch": 0.3861003861003861, + "grad_norm": 1.5354057550430298, + "learning_rate": 0.00015093955950910574, + "loss": 0.8738698577880859, + "mean_token_accuracy": 0.761322969198227, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 0.9465948975086212, + "epoch": 0.5148005148005148, + "grad_norm": 1.3178573846817017, + "learning_rate": 0.00020159041840477876, + "loss": 0.854097671508789, + "mean_token_accuracy": 0.7649031269550324, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9218364179134368, + "epoch": 0.6435006435006435, + "grad_norm": 1.3482104539871216, + "learning_rate": 0.00025224127730045186, + "loss": 0.8339097595214844, + "mean_token_accuracy": 0.7713709837198257, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.92090303003788, + "epoch": 0.7722007722007722, + "grad_norm": 1.1728777885437012, + "learning_rate": 0.00030289213619612493, + "loss": 0.8308000183105468, + "mean_token_accuracy": 0.7717143112421035, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9001683223247529, + "epoch": 0.9009009009009009, + "grad_norm": 1.4331250190734863, + "learning_rate": 0.00035354299509179795, + "loss": 0.8269673919677735, + "mean_token_accuracy": 0.7711574441194534, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7020757987941664, + "eval_loss": 0.8044202923774719, + "eval_mean_token_accuracy": 0.7759947469553996, + "eval_num_tokens": 527146.0, + "eval_runtime": 78.3605, + "eval_samples_per_second": 19.729, + "eval_steps_per_second": 2.476, + "step": 389 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8612565013819008.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9719c9856db8e1cc1178fbbedbbeb71d9052cd46 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.02616102090367193, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0b179af64345dea46ecffd430b87e5462a416b86 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3890/trainer_state.json @@ -0,0 +1,914 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 3890, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.035328435897827, + "epoch": 0.1287001287001287, + "grad_norm": 3.0673413276672363, + "learning_rate": 4.96378417177596e-05, + "loss": 1.9008338928222657, + "mean_token_accuracy": 0.605284059047699, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.0828855693340302, + "epoch": 0.2574002574002574, + "grad_norm": 2.4980359077453613, + "learning_rate": 0.00010028870061343267, + "loss": 0.9762400817871094, + "mean_token_accuracy": 0.7393774157762527, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 0.9727641302347183, + "epoch": 0.3861003861003861, + "grad_norm": 1.5354057550430298, + "learning_rate": 0.00015093955950910574, + "loss": 0.8738698577880859, + "mean_token_accuracy": 0.761322969198227, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 0.9465948975086212, + "epoch": 0.5148005148005148, + "grad_norm": 1.3178573846817017, + "learning_rate": 0.00020159041840477876, + "loss": 0.854097671508789, + "mean_token_accuracy": 0.7649031269550324, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9218364179134368, + "epoch": 0.6435006435006435, + "grad_norm": 1.3482104539871216, + "learning_rate": 0.00025224127730045186, + "loss": 0.8339097595214844, + "mean_token_accuracy": 0.7713709837198257, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.92090303003788, + "epoch": 0.7722007722007722, + "grad_norm": 1.1728777885437012, + "learning_rate": 0.00030289213619612493, + "loss": 0.8308000183105468, + "mean_token_accuracy": 0.7717143112421035, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9001683223247529, + "epoch": 0.9009009009009009, + "grad_norm": 1.4331250190734863, + "learning_rate": 0.00035354299509179795, + "loss": 0.8269673919677735, + "mean_token_accuracy": 0.7711574441194534, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7020757987941664, + "eval_loss": 0.8044202923774719, + "eval_mean_token_accuracy": 0.7759947469553996, + "eval_num_tokens": 527146.0, + "eval_runtime": 78.3605, + "eval_samples_per_second": 19.729, + "eval_steps_per_second": 2.476, + "step": 389 + }, + { + "entropy": 0.8767053924425684, + "epoch": 1.0283140283140284, + "grad_norm": 1.3621660470962524, + "learning_rate": 0.0003940557495448841, + "loss": 0.8080394744873047, + "mean_token_accuracy": 0.7781236532962683, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8415434396266938, + "epoch": 1.157014157014157, + "grad_norm": 1.734777808189392, + "learning_rate": 0.00039377817338715463, + "loss": 0.7779766082763672, + "mean_token_accuracy": 0.7830026543140411, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8639528834819794, + "epoch": 1.2857142857142856, + "grad_norm": 1.3795595169067383, + "learning_rate": 0.00039310460257672286, + "loss": 0.7801459503173828, + "mean_token_accuracy": 0.7819079357385635, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.8538346707820892, + "epoch": 1.4144144144144144, + "grad_norm": 1.6661036014556885, + "learning_rate": 0.00039203639282089, + "loss": 0.7902972412109375, + "mean_token_accuracy": 0.7807946759462356, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8081058472394943, + "epoch": 1.5431145431145432, + "grad_norm": 1.2289175987243652, + "learning_rate": 0.00039057569412340117, + "loss": 0.7647433471679688, + "mean_token_accuracy": 0.7846344596147538, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8334913140535355, + "epoch": 1.6718146718146718, + "grad_norm": 2.2659049034118652, + "learning_rate": 0.0003887254464570972, + "loss": 0.7828218078613282, + "mean_token_accuracy": 0.7819693273305893, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8104432183504104, + "epoch": 1.8005148005148004, + "grad_norm": 1.7065047025680542, + "learning_rate": 0.00038648937384658125, + "loss": 0.763125, + "mean_token_accuracy": 0.7861280244588852, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8062294274568558, + "epoch": 1.9292149292149292, + "grad_norm": 1.3628491163253784, + "learning_rate": 0.0003838719768728124, + "loss": 0.7553546905517579, + "mean_token_accuracy": 0.7892439645528794, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.679631345665332, + "eval_loss": 0.7782689929008484, + "eval_mean_token_accuracy": 0.787303302398662, + "eval_num_tokens": 1054292.0, + "eval_runtime": 75.9942, + "eval_samples_per_second": 20.344, + "eval_steps_per_second": 2.553, + "step": 778 + }, + { + "entropy": 0.749360041786926, + "epoch": 2.056628056628057, + "grad_norm": 1.084079623222351, + "learning_rate": 0.0003808785236147101, + "loss": 0.7160065460205078, + "mean_token_accuracy": 0.7958323576233604, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.6975917857885361, + "epoch": 2.1853281853281854, + "grad_norm": 1.3939169645309448, + "learning_rate": 0.00037751503904600377, + "loss": 0.6710267639160157, + "mean_token_accuracy": 0.8034947836399078, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7072162652015686, + "epoch": 2.314028314028314, + "grad_norm": 1.9786381721496582, + "learning_rate": 0.0003737882929086661, + "loss": 0.6777350616455078, + "mean_token_accuracy": 0.8026440101861954, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7069408267736434, + "epoch": 2.4427284427284426, + "grad_norm": 1.3146620988845825, + "learning_rate": 0.00036970578608734, + "loss": 0.6795073699951172, + "mean_token_accuracy": 0.8013493180274963, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.6993900266289711, + "epoch": 2.571428571428571, + "grad_norm": 1.5064544677734375, + "learning_rate": 0.00036527573551218173, + "loss": 0.6791993713378907, + "mean_token_accuracy": 0.8022604930400848, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.6983578675985336, + "epoch": 2.7001287001287, + "grad_norm": 1.2354060411453247, + "learning_rate": 0.0003605070576205073, + "loss": 0.6693215179443359, + "mean_token_accuracy": 0.8064261078834534, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7105095881223679, + "epoch": 2.828828828828829, + "grad_norm": 1.182181477546692, + "learning_rate": 0.00035540935041052915, + "loss": 0.6762094116210937, + "mean_token_accuracy": 0.8052380502223968, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7148650509119033, + "epoch": 2.9575289575289574, + "grad_norm": 0.9586004018783569, + "learning_rate": 0.00034999287412330383, + "loss": 0.675987777709961, + "mean_token_accuracy": 0.8049957078695297, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6378419605112567, + "eval_loss": 0.7481877207756042, + "eval_mean_token_accuracy": 0.7935938085477376, + "eval_num_tokens": 1581438.0, + "eval_runtime": 75.8368, + "eval_samples_per_second": 20.386, + "eval_steps_per_second": 2.558, + "step": 1167 + }, + { + "entropy": 0.6485739145616088, + "epoch": 3.0849420849420848, + "grad_norm": 1.1198654174804688, + "learning_rate": 0.00034426853059177186, + "loss": 0.6087686157226563, + "mean_token_accuracy": 0.8164294560750326, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6030477187037468, + "epoch": 3.213642213642214, + "grad_norm": 1.1516684293746948, + "learning_rate": 0.00033824784129845556, + "loss": 0.5574645614624023, + "mean_token_accuracy": 0.8271971315145492, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6140900093317032, + "epoch": 3.3423423423423424, + "grad_norm": 1.372568964958191, + "learning_rate": 0.00033194292418597715, + "loss": 0.5769831848144531, + "mean_token_accuracy": 0.8246584522724152, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6223019778728485, + "epoch": 3.471042471042471, + "grad_norm": 1.5307430028915405, + "learning_rate": 0.0003253664692670717, + "loss": 0.5899515914916992, + "mean_token_accuracy": 0.820688226222992, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6215043878555297, + "epoch": 3.5997425997425996, + "grad_norm": 1.487574577331543, + "learning_rate": 0.0003185317130831854, + "loss": 0.5944060134887695, + "mean_token_accuracy": 0.8215733963251114, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6398005878925324, + "epoch": 3.7284427284427286, + "grad_norm": 1.054810881614685, + "learning_rate": 0.0003114524120630655, + "loss": 0.5942705154418946, + "mean_token_accuracy": 0.8215270626544953, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6384156176447868, + "epoch": 3.857142857142857, + "grad_norm": 1.5161454677581787, + "learning_rate": 0.00030414281483496505, + "loss": 0.5947936630249023, + "mean_token_accuracy": 0.8206489557027816, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.598966498374939, + "epoch": 3.985842985842986, + "grad_norm": 1.2136448621749878, + "learning_rate": 0.00029661763354818884, + "loss": 0.5746045684814454, + "mean_token_accuracy": 0.8274296247959136, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5822035986738107, + "eval_loss": 0.7569039463996887, + "eval_mean_token_accuracy": 0.7907421751735136, + "eval_num_tokens": 2108584.0, + "eval_runtime": 76.165, + "eval_samples_per_second": 20.298, + "eval_steps_per_second": 2.547, + "step": 1556 + }, + { + "entropy": 0.5126810152121265, + "epoch": 4.113256113256114, + "grad_norm": 1.2219198942184448, + "learning_rate": 0.00028889201426170327, + "loss": 0.4712484359741211, + "mean_token_accuracy": 0.8507941756585632, + "num_tokens": 2165782.0, + "step": 1600 + }, + { + "entropy": 0.5017523977160454, + "epoch": 4.241956241956242, + "grad_norm": 1.1246048212051392, + "learning_rate": 0.00028098150645940856, + "loss": 0.45758853912353514, + "mean_token_accuracy": 0.8549614226818085, + "num_tokens": 2233206.0, + "step": 1650 + }, + { + "entropy": 0.5128893256187439, + "epoch": 4.370656370656371, + "grad_norm": 1.2206052541732788, + "learning_rate": 0.000272902031753431, + "loss": 0.4767184066772461, + "mean_token_accuracy": 0.8486203062534332, + "num_tokens": 2301399.0, + "step": 1700 + }, + { + "entropy": 0.500448043346405, + "epoch": 4.499356499356499, + "grad_norm": 1.099212884902954, + "learning_rate": 0.0002646698518384262, + "loss": 0.46724525451660154, + "mean_token_accuracy": 0.8502935725450516, + "num_tokens": 2370831.0, + "step": 1750 + }, + { + "entropy": 0.5091731676459312, + "epoch": 4.628056628056628, + "grad_norm": 1.0495911836624146, + "learning_rate": 0.0002563015357613929, + "loss": 0.4759058380126953, + "mean_token_accuracy": 0.8488407224416733, + "num_tokens": 2439168.0, + "step": 1800 + }, + { + "entropy": 0.515411552786827, + "epoch": 4.756756756756757, + "grad_norm": 0.9209436774253845, + "learning_rate": 0.00024781392657287327, + "loss": 0.48281356811523435, + "mean_token_accuracy": 0.8456312155723572, + "num_tokens": 2508036.0, + "step": 1850 + }, + { + "entropy": 0.5016480022668839, + "epoch": 4.885456885456885, + "grad_norm": 1.5305742025375366, + "learning_rate": 0.00023922410742666182, + "loss": 0.4707856369018555, + "mean_token_accuracy": 0.8486748856306076, + "num_tokens": 2577025.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5264220541899967, + "eval_loss": 0.7557247281074524, + "eval_mean_token_accuracy": 0.7990376009154565, + "eval_num_tokens": 2635730.0, + "eval_runtime": 76.2827, + "eval_samples_per_second": 20.267, + "eval_steps_per_second": 2.543, + "step": 1945 + }, + { + "entropy": 0.5138596534126937, + "epoch": 5.012870012870013, + "grad_norm": 0.7728239893913269, + "learning_rate": 0.00023054936719625412, + "loss": 0.4731882095336914, + "mean_token_accuracy": 0.8493862681918674, + "num_tokens": 2642803.0, + "step": 1950 + }, + { + "entropy": 0.3956746557354927, + "epoch": 5.141570141570142, + "grad_norm": 0.852727472782135, + "learning_rate": 0.00022180716567724064, + "loss": 0.3441731262207031, + "mean_token_accuracy": 0.8864887124300003, + "num_tokens": 2708703.0, + "step": 2000 + }, + { + "entropy": 0.3931361359357834, + "epoch": 5.27027027027027, + "grad_norm": 1.062345266342163, + "learning_rate": 0.00021301509844568218, + "loss": 0.34529010772705077, + "mean_token_accuracy": 0.88379523396492, + "num_tokens": 2777956.0, + "step": 2050 + }, + { + "entropy": 0.3996828556060791, + "epoch": 5.398970398970399, + "grad_norm": 1.1475750207901, + "learning_rate": 0.00020419086144319811, + "loss": 0.35055686950683596, + "mean_token_accuracy": 0.8822557419538498, + "num_tokens": 2847032.0, + "step": 2100 + }, + { + "entropy": 0.3957197970151901, + "epoch": 5.527670527670527, + "grad_norm": 0.9748324751853943, + "learning_rate": 0.00019535221536004678, + "loss": 0.3523390197753906, + "mean_token_accuracy": 0.8818045473098755, + "num_tokens": 2915746.0, + "step": 2150 + }, + { + "entropy": 0.39835822969675067, + "epoch": 5.656370656370656, + "grad_norm": 1.1422828435897827, + "learning_rate": 0.00018651694988788578, + "loss": 0.35444053649902346, + "mean_token_accuracy": 0.8799562251567841, + "num_tokens": 2984346.0, + "step": 2200 + }, + { + "entropy": 0.40228330612182617, + "epoch": 5.785070785070785, + "grad_norm": 0.9757609963417053, + "learning_rate": 0.0001777028479141599, + "loss": 0.3603257751464844, + "mean_token_accuracy": 0.8790501236915589, + "num_tokens": 3051863.0, + "step": 2250 + }, + { + "entropy": 0.39846932888031006, + "epoch": 5.913770913770914, + "grad_norm": 1.058344841003418, + "learning_rate": 0.00016892764973018475, + "loss": 0.3548299407958984, + "mean_token_accuracy": 0.8812332856655121, + "num_tokens": 3119763.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4225990093245949, + "eval_loss": 0.8478386402130127, + "eval_mean_token_accuracy": 0.7951162933688802, + "eval_num_tokens": 3162876.0, + "eval_runtime": 76.4757, + "eval_samples_per_second": 20.216, + "eval_steps_per_second": 2.537, + "step": 2334 + }, + { + "entropy": 0.35978798884333985, + "epoch": 6.041184041184041, + "grad_norm": 1.1438041925430298, + "learning_rate": 0.00016020901732496375, + "loss": 0.3136854934692383, + "mean_token_accuracy": 0.8952204982439677, + "num_tokens": 3185495.0, + "step": 2350 + }, + { + "entropy": 0.274916353225708, + "epoch": 6.1698841698841695, + "grad_norm": 1.063818097114563, + "learning_rate": 0.000151564498836606, + "loss": 0.22996776580810546, + "mean_token_accuracy": 0.9221211689710617, + "num_tokens": 3252659.0, + "step": 2400 + }, + { + "entropy": 0.27972787141799926, + "epoch": 6.298584298584299, + "grad_norm": 0.9946160316467285, + "learning_rate": 0.00014301149323289332, + "loss": 0.23536941528320313, + "mean_token_accuracy": 0.9205723887681961, + "num_tokens": 3320155.0, + "step": 2450 + }, + { + "entropy": 0.27985903441905974, + "epoch": 6.427284427284428, + "grad_norm": 1.090388536453247, + "learning_rate": 0.00013456721529208496, + "loss": 0.23937501907348632, + "mean_token_accuracy": 0.9185256743431092, + "num_tokens": 3384147.0, + "step": 2500 + }, + { + "entropy": 0.27657748594880105, + "epoch": 6.555984555984556, + "grad_norm": 0.9090324640274048, + "learning_rate": 0.00012624866095444425, + "loss": 0.23602998733520508, + "mean_token_accuracy": 0.9194209152460098, + "num_tokens": 3453826.0, + "step": 2550 + }, + { + "entropy": 0.28343546360731126, + "epoch": 6.684684684684685, + "grad_norm": 0.956750750541687, + "learning_rate": 0.00011807257311422335, + "loss": 0.24055482864379882, + "mean_token_accuracy": 0.9179503041505813, + "num_tokens": 3521099.0, + "step": 2600 + }, + { + "entropy": 0.2764540535211563, + "epoch": 6.813384813384813, + "grad_norm": 0.9264359474182129, + "learning_rate": 0.0001100554079209585, + "loss": 0.23727087020874024, + "mean_token_accuracy": 0.9196303296089172, + "num_tokens": 3588515.0, + "step": 2650 + }, + { + "entropy": 0.27410337537527085, + "epoch": 6.942084942084942, + "grad_norm": 0.9057352542877197, + "learning_rate": 0.00010221330165790096, + "loss": 0.23279956817626954, + "mean_token_accuracy": 0.9197283333539963, + "num_tokens": 3658319.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.3525028199571924, + "eval_loss": 0.9255094528198242, + "eval_mean_token_accuracy": 0.7976236954792258, + "eval_num_tokens": 3690022.0, + "eval_runtime": 76.074, + "eval_samples_per_second": 20.322, + "eval_steps_per_second": 2.55, + "step": 2723 + }, + { + "entropy": 0.22481059982921137, + "epoch": 7.06949806949807, + "grad_norm": 1.1758164167404175, + "learning_rate": 9.456203826424829e-05, + "loss": 0.1848788833618164, + "mean_token_accuracy": 0.9379562127469766, + "num_tokens": 3727811.0, + "step": 2750 + }, + { + "entropy": 0.18282695084810258, + "epoch": 7.198198198198198, + "grad_norm": 0.781952977180481, + "learning_rate": 8.711701756654426e-05, + "loss": 0.1484829616546631, + "mean_token_accuracy": 0.9501194298267365, + "num_tokens": 3795920.0, + "step": 2800 + }, + { + "entropy": 0.18234790086746216, + "epoch": 7.326898326898327, + "grad_norm": 0.703240692615509, + "learning_rate": 7.989322428318929e-05, + "loss": 0.15111746788024902, + "mean_token_accuracy": 0.9502605730295182, + "num_tokens": 3863324.0, + "step": 2850 + }, + { + "entropy": 0.1816841357946396, + "epoch": 7.455598455598455, + "grad_norm": 1.0166149139404297, + "learning_rate": 7.290519786444519e-05, + "loss": 0.15027329444885254, + "mean_token_accuracy": 0.9499351298809051, + "num_tokens": 3932191.0, + "step": 2900 + }, + { + "entropy": 0.17630216613411903, + "epoch": 7.584298584298584, + "grad_norm": 0.5356483459472656, + "learning_rate": 6.616700322863927e-05, + "loss": 0.14939964294433594, + "mean_token_accuracy": 0.9505619239807129, + "num_tokens": 4000632.0, + "step": 2950 + }, + { + "entropy": 0.18283376961946488, + "epoch": 7.712998712998713, + "grad_norm": 0.7002793550491333, + "learning_rate": 5.969220245346672e-05, + "loss": 0.15329421997070314, + "mean_token_accuracy": 0.949188598394394, + "num_tokens": 4067753.0, + "step": 3000 + }, + { + "entropy": 0.18244779869914055, + "epoch": 7.841698841698841, + "grad_norm": 0.6270423531532288, + "learning_rate": 5.349382747936841e-05, + "loss": 0.15302456855773927, + "mean_token_accuracy": 0.9504549181461335, + "num_tokens": 4133681.0, + "step": 3050 + }, + { + "entropy": 0.17880768597126007, + "epoch": 7.97039897039897, + "grad_norm": 0.6172229647636414, + "learning_rate": 4.758435387992531e-05, + "loss": 0.14883087158203126, + "mean_token_accuracy": 0.9509924799203873, + "num_tokens": 4201263.0, + "step": 3100 + }, + { + "epoch": 8.0, + "eval_entropy": 0.28235290898490195, + "eval_loss": 1.1443017721176147, + "eval_mean_token_accuracy": 0.7958696088225571, + "eval_num_tokens": 4217168.0, + "eval_runtime": 76.1084, + "eval_samples_per_second": 20.313, + "eval_steps_per_second": 2.549, + "step": 3112 + }, + { + "entropy": 0.15051050589542198, + "epoch": 8.097812097812097, + "grad_norm": 0.4672251045703888, + "learning_rate": 4.197567575206164e-05, + "loss": 0.12156478881835937, + "mean_token_accuracy": 0.9611766500906511, + "num_tokens": 4268387.0, + "step": 3150 + }, + { + "entropy": 0.13742553241550923, + "epoch": 8.226512226512227, + "grad_norm": 0.7555282115936279, + "learning_rate": 3.667908177659604e-05, + "loss": 0.11531492233276368, + "mean_token_accuracy": 0.9618133813142776, + "num_tokens": 4335443.0, + "step": 3200 + }, + { + "entropy": 0.13783976405858994, + "epoch": 8.355212355212355, + "grad_norm": 0.372898131608963, + "learning_rate": 3.170523249732349e-05, + "loss": 0.11172324180603027, + "mean_token_accuracy": 0.9617116987705231, + "num_tokens": 4405738.0, + "step": 3250 + }, + { + "entropy": 0.1400664184987545, + "epoch": 8.483912483912484, + "grad_norm": 0.6359309554100037, + "learning_rate": 2.7064138864359793e-05, + "loss": 0.1197418212890625, + "mean_token_accuracy": 0.9595164811611175, + "num_tokens": 4471215.0, + "step": 3300 + }, + { + "entropy": 0.1393050680309534, + "epoch": 8.612612612612612, + "grad_norm": 0.47000861167907715, + "learning_rate": 2.2765142084933584e-05, + "loss": 0.11754611015319824, + "mean_token_accuracy": 0.9604016053676605, + "num_tokens": 4538643.0, + "step": 3350 + }, + { + "entropy": 0.14121561706066132, + "epoch": 8.741312741312742, + "grad_norm": 0.6012074947357178, + "learning_rate": 1.8816894822181725e-05, + "loss": 0.12010603904724121, + "mean_token_accuracy": 0.9601105463504791, + "num_tokens": 4604547.0, + "step": 3400 + }, + { + "entropy": 0.13598045460879804, + "epoch": 8.87001287001287, + "grad_norm": 0.44798487424850464, + "learning_rate": 1.522734377978824e-05, + "loss": 0.1126248836517334, + "mean_token_accuracy": 0.9618371516466141, + "num_tokens": 4675035.0, + "step": 3450 + }, + { + "entropy": 0.1365610294044018, + "epoch": 8.998712998712998, + "grad_norm": 0.3193477690219879, + "learning_rate": 1.2003713707520592e-05, + "loss": 0.11523623466491699, + "mean_token_accuracy": 0.961252732872963, + "num_tokens": 4743741.0, + "step": 3500 + }, + { + "epoch": 9.0, + "eval_entropy": 0.256610156641793, + "eval_loss": 1.2551246881484985, + "eval_mean_token_accuracy": 0.7978993546716946, + "eval_num_tokens": 4744314.0, + "eval_runtime": 77.606, + "eval_samples_per_second": 19.921, + "eval_steps_per_second": 2.5, + "step": 3501 + }, + { + "entropy": 0.13025417126188374, + "epoch": 9.126126126126126, + "grad_norm": 0.3649803102016449, + "learning_rate": 9.152492859853847e-06, + "loss": 0.10353655815124511, + "mean_token_accuracy": 0.9643954038619995, + "num_tokens": 4810493.0, + "step": 3550 + }, + { + "entropy": 0.12697464868426322, + "epoch": 9.254826254826256, + "grad_norm": 0.298749178647995, + "learning_rate": 6.679419936952161e-06, + "loss": 0.1044382095336914, + "mean_token_accuracy": 0.965098876953125, + "num_tokens": 4877111.0, + "step": 3600 + }, + { + "entropy": 0.12468758068978786, + "epoch": 9.383526383526384, + "grad_norm": 0.2773562967777252, + "learning_rate": 4.589472534290043e-06, + "loss": 0.10384930610656738, + "mean_token_accuracy": 0.9643976593017578, + "num_tokens": 4945117.0, + "step": 3650 + }, + { + "entropy": 0.12696147769689559, + "epoch": 9.512226512226512, + "grad_norm": 0.29385918378829956, + "learning_rate": 2.886857124162505e-06, + "loss": 0.10573678016662598, + "mean_token_accuracy": 0.9632026982307434, + "num_tokens": 5012749.0, + "step": 3700 + }, + { + "entropy": 0.12652831569314002, + "epoch": 9.64092664092664, + "grad_norm": 0.43310844898223877, + "learning_rate": 1.5750005892472825e-06, + "loss": 0.10498411178588868, + "mean_token_accuracy": 0.9637316900491715, + "num_tokens": 5080092.0, + "step": 3750 + }, + { + "entropy": 0.12831369310617446, + "epoch": 9.76962676962677, + "grad_norm": 0.3614860773086548, + "learning_rate": 6.565433252606269e-07, + "loss": 0.10618659973144531, + "mean_token_accuracy": 0.9635689997673035, + "num_tokens": 5147114.0, + "step": 3800 + }, + { + "entropy": 0.12164834998548031, + "epoch": 9.898326898326898, + "grad_norm": 0.3508666753768921, + "learning_rate": 1.3333392658824524e-07, + "loss": 0.10175061225891113, + "mean_token_accuracy": 0.965265138745308, + "num_tokens": 5217747.0, + "step": 3850 + }, + { + "epoch": 10.0, + "eval_entropy": 0.25325659270753564, + "eval_loss": 1.3216900825500488, + "eval_mean_token_accuracy": 0.797172186608167, + "eval_num_tokens": 5271460.0, + "eval_runtime": 76.1184, + "eval_samples_per_second": 20.31, + "eval_steps_per_second": 2.549, + "step": 3890 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 8.619941744738573e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9719c9856db8e1cc1178fbbedbbeb71d9052cd46 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.02616102090367193, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ce7a926abb4b50f96486d623d6ea4bd8bad2a0b1 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-778/trainer_state.json @@ -0,0 +1,206 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 778, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.035328435897827, + "epoch": 0.1287001287001287, + "grad_norm": 3.0673413276672363, + "learning_rate": 4.96378417177596e-05, + "loss": 1.9008338928222657, + "mean_token_accuracy": 0.605284059047699, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.0828855693340302, + "epoch": 0.2574002574002574, + "grad_norm": 2.4980359077453613, + "learning_rate": 0.00010028870061343267, + "loss": 0.9762400817871094, + "mean_token_accuracy": 0.7393774157762527, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 0.9727641302347183, + "epoch": 0.3861003861003861, + "grad_norm": 1.5354057550430298, + "learning_rate": 0.00015093955950910574, + "loss": 0.8738698577880859, + "mean_token_accuracy": 0.761322969198227, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 0.9465948975086212, + "epoch": 0.5148005148005148, + "grad_norm": 1.3178573846817017, + "learning_rate": 0.00020159041840477876, + "loss": 0.854097671508789, + "mean_token_accuracy": 0.7649031269550324, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9218364179134368, + "epoch": 0.6435006435006435, + "grad_norm": 1.3482104539871216, + "learning_rate": 0.00025224127730045186, + "loss": 0.8339097595214844, + "mean_token_accuracy": 0.7713709837198257, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.92090303003788, + "epoch": 0.7722007722007722, + "grad_norm": 1.1728777885437012, + "learning_rate": 0.00030289213619612493, + "loss": 0.8308000183105468, + "mean_token_accuracy": 0.7717143112421035, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9001683223247529, + "epoch": 0.9009009009009009, + "grad_norm": 1.4331250190734863, + "learning_rate": 0.00035354299509179795, + "loss": 0.8269673919677735, + "mean_token_accuracy": 0.7711574441194534, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7020757987941664, + "eval_loss": 0.8044202923774719, + "eval_mean_token_accuracy": 0.7759947469553996, + "eval_num_tokens": 527146.0, + "eval_runtime": 78.3605, + "eval_samples_per_second": 19.729, + "eval_steps_per_second": 2.476, + "step": 389 + }, + { + "entropy": 0.8767053924425684, + "epoch": 1.0283140283140284, + "grad_norm": 1.3621660470962524, + "learning_rate": 0.0003940557495448841, + "loss": 0.8080394744873047, + "mean_token_accuracy": 0.7781236532962683, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8415434396266938, + "epoch": 1.157014157014157, + "grad_norm": 1.734777808189392, + "learning_rate": 0.00039377817338715463, + "loss": 0.7779766082763672, + "mean_token_accuracy": 0.7830026543140411, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8639528834819794, + "epoch": 1.2857142857142856, + "grad_norm": 1.3795595169067383, + "learning_rate": 0.00039310460257672286, + "loss": 0.7801459503173828, + "mean_token_accuracy": 0.7819079357385635, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.8538346707820892, + "epoch": 1.4144144144144144, + "grad_norm": 1.6661036014556885, + "learning_rate": 0.00039203639282089, + "loss": 0.7902972412109375, + "mean_token_accuracy": 0.7807946759462356, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8081058472394943, + "epoch": 1.5431145431145432, + "grad_norm": 1.2289175987243652, + "learning_rate": 0.00039057569412340117, + "loss": 0.7647433471679688, + "mean_token_accuracy": 0.7846344596147538, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8334913140535355, + "epoch": 1.6718146718146718, + "grad_norm": 2.2659049034118652, + "learning_rate": 0.0003887254464570972, + "loss": 0.7828218078613282, + "mean_token_accuracy": 0.7819693273305893, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8104432183504104, + "epoch": 1.8005148005148004, + "grad_norm": 1.7065047025680542, + "learning_rate": 0.00038648937384658125, + "loss": 0.763125, + "mean_token_accuracy": 0.7861280244588852, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8062294274568558, + "epoch": 1.9292149292149292, + "grad_norm": 1.3628491163253784, + "learning_rate": 0.0003838719768728124, + "loss": 0.7553546905517579, + "mean_token_accuracy": 0.7892439645528794, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.679631345665332, + "eval_loss": 0.7782689929008484, + "eval_mean_token_accuracy": 0.787303302398662, + "eval_num_tokens": 1054292.0, + "eval_runtime": 75.9942, + "eval_samples_per_second": 20.344, + "eval_steps_per_second": 2.553, + "step": 778 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.7207787016141056e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9caa145c2e89237436505b6b8444ecece2f29c63 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/trainer_state.json @@ -0,0 +1,388 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1556, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.936105194091797, + "epoch": 0.1287001287001287, + "grad_norm": 2.5512146949768066, + "learning_rate": 7.410977855063067e-06, + "loss": 2.765989074707031, + "mean_token_accuracy": 0.5075639402866363, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.4886422049999237, + "epoch": 0.2574002574002574, + "grad_norm": 2.2254061698913574, + "learning_rate": 1.4973200156147831e-05, + "loss": 1.336165313720703, + "mean_token_accuracy": 0.6740516322851181, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 1.1471627122163772, + "epoch": 0.3861003861003861, + "grad_norm": 2.2073140144348145, + "learning_rate": 2.2535422457232594e-05, + "loss": 1.0278042602539061, + "mean_token_accuracy": 0.7280837368965148, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 1.0486015713214873, + "epoch": 0.5148005148005148, + "grad_norm": 1.650594711303711, + "learning_rate": 3.0097644758317354e-05, + "loss": 0.933094253540039, + "mean_token_accuracy": 0.7478678917884827, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9942852139472962, + "epoch": 0.6435006435006435, + "grad_norm": 1.7949399948120117, + "learning_rate": 3.765986705940212e-05, + "loss": 0.8747442626953125, + "mean_token_accuracy": 0.7603094267845154, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.9491432571411133, + "epoch": 0.7722007722007722, + "grad_norm": 1.33268404006958, + "learning_rate": 4.522208936048689e-05, + "loss": 0.8404794311523438, + "mean_token_accuracy": 0.7671167278289794, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9246599394083023, + "epoch": 0.9009009009009009, + "grad_norm": 1.3137660026550293, + "learning_rate": 5.278431166157165e-05, + "loss": 0.8163955688476563, + "mean_token_accuracy": 0.7712420308589936, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7913847528782088, + "eval_loss": 0.7582769393920898, + "eval_mean_token_accuracy": 0.7787086770092089, + "eval_num_tokens": 527146.0, + "eval_runtime": 80.7434, + "eval_samples_per_second": 19.135, + "eval_steps_per_second": 2.403, + "step": 389 + }, + { + "entropy": 0.9051812700550965, + "epoch": 1.0283140283140284, + "grad_norm": 1.3198363780975342, + "learning_rate": 5.883290514810127e-05, + "loss": 0.7887770843505859, + "mean_token_accuracy": 0.7787209345836832, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8626887363195419, + "epoch": 1.157014157014157, + "grad_norm": 1.1772489547729492, + "learning_rate": 5.8791462758850675e-05, + "loss": 0.7505465698242187, + "mean_token_accuracy": 0.7887020426988601, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8474252647161484, + "epoch": 1.2857142857142856, + "grad_norm": 1.0174403190612793, + "learning_rate": 5.8690897984332274e-05, + "loss": 0.7397904968261719, + "mean_token_accuracy": 0.7890202105045319, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.843030309677124, + "epoch": 1.4144144144144144, + "grad_norm": 1.1678341627120972, + "learning_rate": 5.853141323295946e-05, + "loss": 0.7369876098632813, + "mean_token_accuracy": 0.790421934723854, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8115463078022003, + "epoch": 1.5431145431145432, + "grad_norm": 0.8854549527168274, + "learning_rate": 5.831332950237421e-05, + "loss": 0.7081725311279297, + "mean_token_accuracy": 0.7952603316307068, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8311941117048264, + "epoch": 1.6718146718146718, + "grad_norm": 1.171217441558838, + "learning_rate": 5.803708573336975e-05, + "loss": 0.7232355499267578, + "mean_token_accuracy": 0.7947697073221207, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8120440804958343, + "epoch": 1.8005148005148004, + "grad_norm": 1.0333378314971924, + "learning_rate": 5.770323792642697e-05, + "loss": 0.6972620391845703, + "mean_token_accuracy": 0.7987716430425644, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8045441079139709, + "epoch": 1.9292149292149292, + "grad_norm": 1.01642906665802, + "learning_rate": 5.7312458022642996e-05, + "loss": 0.6930023193359375, + "mean_token_accuracy": 0.8009403163194656, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6984818912658495, + "eval_loss": 0.6775499582290649, + "eval_mean_token_accuracy": 0.8020080097557343, + "eval_num_tokens": 1054292.0, + "eval_runtime": 78.3908, + "eval_samples_per_second": 19.709, + "eval_steps_per_second": 2.475, + "step": 778 + }, + { + "entropy": 0.7779398566544659, + "epoch": 2.056628056628057, + "grad_norm": 1.4157040119171143, + "learning_rate": 5.686553255130396e-05, + "loss": 0.6584475708007812, + "mean_token_accuracy": 0.8087991054611977, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.745064823627472, + "epoch": 2.1853281853281854, + "grad_norm": 0.9885228872299194, + "learning_rate": 5.636336104682433e-05, + "loss": 0.618268051147461, + "mean_token_accuracy": 0.8165298491716385, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7321632200479508, + "epoch": 2.314028314028314, + "grad_norm": 1.0271121263504028, + "learning_rate": 5.580695423823882e-05, + "loss": 0.6195047760009765, + "mean_token_accuracy": 0.8162743961811065, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7240813142061233, + "epoch": 2.4427284427284426, + "grad_norm": 0.8744781613349915, + "learning_rate": 5.519743201489109e-05, + "loss": 0.6093913650512696, + "mean_token_accuracy": 0.8179467821121216, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.7273010468482971, + "epoch": 2.571428571428571, + "grad_norm": 1.0161406993865967, + "learning_rate": 5.453602117241361e-05, + "loss": 0.6118500518798828, + "mean_token_accuracy": 0.8183792388439178, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.7150553166866302, + "epoch": 2.7001287001287, + "grad_norm": 0.9795015454292297, + "learning_rate": 5.382405294353544e-05, + "loss": 0.6005525970458985, + "mean_token_accuracy": 0.8210368871688842, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7134475702047348, + "epoch": 2.828828828828829, + "grad_norm": 0.9153344035148621, + "learning_rate": 5.306296031868776e-05, + "loss": 0.6033024978637695, + "mean_token_accuracy": 0.8216253930330276, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7142637860774994, + "epoch": 2.9575289575289574, + "grad_norm": 0.8638553619384766, + "learning_rate": 5.2254275161799905e-05, + "loss": 0.6056064605712891, + "mean_token_accuracy": 0.8204184120893478, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6126322589584232, + "eval_loss": 0.676252007484436, + "eval_mean_token_accuracy": 0.8002185584958067, + "eval_num_tokens": 1581438.0, + "eval_runtime": 78.2302, + "eval_samples_per_second": 19.749, + "eval_steps_per_second": 2.48, + "step": 1167 + }, + { + "entropy": 0.6793721679485205, + "epoch": 3.0849420849420848, + "grad_norm": 1.2363271713256836, + "learning_rate": 5.139962512709103e-05, + "loss": 0.5553075408935547, + "mean_token_accuracy": 0.8319492520708026, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6361582756042481, + "epoch": 3.213642213642214, + "grad_norm": 1.0183900594711304, + "learning_rate": 5.050073038306314e-05, + "loss": 0.5097637939453125, + "mean_token_accuracy": 0.8410317724943162, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6419171839952469, + "epoch": 3.3423423423423424, + "grad_norm": 1.3186038732528687, + "learning_rate": 4.955940015028897e-05, + "loss": 0.5204424285888671, + "mean_token_accuracy": 0.8418322193622589, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6460894507169723, + "epoch": 3.471042471042471, + "grad_norm": 1.401528000831604, + "learning_rate": 4.857752905996332e-05, + "loss": 0.5238306427001953, + "mean_token_accuracy": 0.8391665583848953, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6538929349184036, + "epoch": 3.5997425997425996, + "grad_norm": 1.3003088235855103, + "learning_rate": 4.7557093340546974e-05, + "loss": 0.5259903335571289, + "mean_token_accuracy": 0.8376457345485687, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6378953090310097, + "epoch": 3.7284427284427286, + "grad_norm": 1.0874862670898438, + "learning_rate": 4.650014684017843e-05, + "loss": 0.5255831527709961, + "mean_token_accuracy": 0.8387808799743652, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6492014318704605, + "epoch": 3.857142857142857, + "grad_norm": 1.2991198301315308, + "learning_rate": 4.54088168928592e-05, + "loss": 0.5239232635498047, + "mean_token_accuracy": 0.8401105499267578, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.6277376067638397, + "epoch": 3.985842985842986, + "grad_norm": 1.1950165033340454, + "learning_rate": 4.428530003673287e-05, + "loss": 0.5076271057128906, + "mean_token_accuracy": 0.8449398750066757, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5833789521885901, + "eval_loss": 0.685325026512146, + "eval_mean_token_accuracy": 0.8025367401309849, + "eval_num_tokens": 2108584.0, + "eval_runtime": 77.1217, + "eval_samples_per_second": 20.033, + "eval_steps_per_second": 2.516, + "step": 1556 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.553872080039616e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f298f9d53609598cf56ba3b881c57739258a0df9 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.06250292706368335, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..faa09786e5140cd4c02647056f1813f7c70946e3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1945/trainer_state.json @@ -0,0 +1,469 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 1945, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.936105194091797, + "epoch": 0.1287001287001287, + "grad_norm": 2.5512146949768066, + "learning_rate": 7.410977855063067e-06, + "loss": 2.765989074707031, + "mean_token_accuracy": 0.5075639402866363, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.4886422049999237, + "epoch": 0.2574002574002574, + "grad_norm": 2.2254061698913574, + "learning_rate": 1.4973200156147831e-05, + "loss": 1.336165313720703, + "mean_token_accuracy": 0.6740516322851181, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 1.1471627122163772, + "epoch": 0.3861003861003861, + "grad_norm": 2.2073140144348145, + "learning_rate": 2.2535422457232594e-05, + "loss": 1.0278042602539061, + "mean_token_accuracy": 0.7280837368965148, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 1.0486015713214873, + "epoch": 0.5148005148005148, + "grad_norm": 1.650594711303711, + "learning_rate": 3.0097644758317354e-05, + "loss": 0.933094253540039, + "mean_token_accuracy": 0.7478678917884827, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9942852139472962, + "epoch": 0.6435006435006435, + "grad_norm": 1.7949399948120117, + "learning_rate": 3.765986705940212e-05, + "loss": 0.8747442626953125, + "mean_token_accuracy": 0.7603094267845154, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.9491432571411133, + "epoch": 0.7722007722007722, + "grad_norm": 1.33268404006958, + "learning_rate": 4.522208936048689e-05, + "loss": 0.8404794311523438, + "mean_token_accuracy": 0.7671167278289794, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9246599394083023, + "epoch": 0.9009009009009009, + "grad_norm": 1.3137660026550293, + "learning_rate": 5.278431166157165e-05, + "loss": 0.8163955688476563, + "mean_token_accuracy": 0.7712420308589936, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7913847528782088, + "eval_loss": 0.7582769393920898, + "eval_mean_token_accuracy": 0.7787086770092089, + "eval_num_tokens": 527146.0, + "eval_runtime": 80.7434, + "eval_samples_per_second": 19.135, + "eval_steps_per_second": 2.403, + "step": 389 + }, + { + "entropy": 0.9051812700550965, + "epoch": 1.0283140283140284, + "grad_norm": 1.3198363780975342, + "learning_rate": 5.883290514810127e-05, + "loss": 0.7887770843505859, + "mean_token_accuracy": 0.7787209345836832, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8626887363195419, + "epoch": 1.157014157014157, + "grad_norm": 1.1772489547729492, + "learning_rate": 5.8791462758850675e-05, + "loss": 0.7505465698242187, + "mean_token_accuracy": 0.7887020426988601, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8474252647161484, + "epoch": 1.2857142857142856, + "grad_norm": 1.0174403190612793, + "learning_rate": 5.8690897984332274e-05, + "loss": 0.7397904968261719, + "mean_token_accuracy": 0.7890202105045319, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.843030309677124, + "epoch": 1.4144144144144144, + "grad_norm": 1.1678341627120972, + "learning_rate": 5.853141323295946e-05, + "loss": 0.7369876098632813, + "mean_token_accuracy": 0.790421934723854, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8115463078022003, + "epoch": 1.5431145431145432, + "grad_norm": 0.8854549527168274, + "learning_rate": 5.831332950237421e-05, + "loss": 0.7081725311279297, + "mean_token_accuracy": 0.7952603316307068, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8311941117048264, + "epoch": 1.6718146718146718, + "grad_norm": 1.171217441558838, + "learning_rate": 5.803708573336975e-05, + "loss": 0.7232355499267578, + "mean_token_accuracy": 0.7947697073221207, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8120440804958343, + "epoch": 1.8005148005148004, + "grad_norm": 1.0333378314971924, + "learning_rate": 5.770323792642697e-05, + "loss": 0.6972620391845703, + "mean_token_accuracy": 0.7987716430425644, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8045441079139709, + "epoch": 1.9292149292149292, + "grad_norm": 1.01642906665802, + "learning_rate": 5.7312458022642996e-05, + "loss": 0.6930023193359375, + "mean_token_accuracy": 0.8009403163194656, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6984818912658495, + "eval_loss": 0.6775499582290649, + "eval_mean_token_accuracy": 0.8020080097557343, + "eval_num_tokens": 1054292.0, + "eval_runtime": 78.3908, + "eval_samples_per_second": 19.709, + "eval_steps_per_second": 2.475, + "step": 778 + }, + { + "entropy": 0.7779398566544659, + "epoch": 2.056628056628057, + "grad_norm": 1.4157040119171143, + "learning_rate": 5.686553255130396e-05, + "loss": 0.6584475708007812, + "mean_token_accuracy": 0.8087991054611977, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.745064823627472, + "epoch": 2.1853281853281854, + "grad_norm": 0.9885228872299194, + "learning_rate": 5.636336104682433e-05, + "loss": 0.618268051147461, + "mean_token_accuracy": 0.8165298491716385, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7321632200479508, + "epoch": 2.314028314028314, + "grad_norm": 1.0271121263504028, + "learning_rate": 5.580695423823882e-05, + "loss": 0.6195047760009765, + "mean_token_accuracy": 0.8162743961811065, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7240813142061233, + "epoch": 2.4427284427284426, + "grad_norm": 0.8744781613349915, + "learning_rate": 5.519743201489109e-05, + "loss": 0.6093913650512696, + "mean_token_accuracy": 0.8179467821121216, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.7273010468482971, + "epoch": 2.571428571428571, + "grad_norm": 1.0161406993865967, + "learning_rate": 5.453602117241361e-05, + "loss": 0.6118500518798828, + "mean_token_accuracy": 0.8183792388439178, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.7150553166866302, + "epoch": 2.7001287001287, + "grad_norm": 0.9795015454292297, + "learning_rate": 5.382405294353544e-05, + "loss": 0.6005525970458985, + "mean_token_accuracy": 0.8210368871688842, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7134475702047348, + "epoch": 2.828828828828829, + "grad_norm": 0.9153344035148621, + "learning_rate": 5.306296031868776e-05, + "loss": 0.6033024978637695, + "mean_token_accuracy": 0.8216253930330276, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7142637860774994, + "epoch": 2.9575289575289574, + "grad_norm": 0.8638553619384766, + "learning_rate": 5.2254275161799905e-05, + "loss": 0.6056064605712891, + "mean_token_accuracy": 0.8204184120893478, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6126322589584232, + "eval_loss": 0.676252007484436, + "eval_mean_token_accuracy": 0.8002185584958067, + "eval_num_tokens": 1581438.0, + "eval_runtime": 78.2302, + "eval_samples_per_second": 19.749, + "eval_steps_per_second": 2.48, + "step": 1167 + }, + { + "entropy": 0.6793721679485205, + "epoch": 3.0849420849420848, + "grad_norm": 1.2363271713256836, + "learning_rate": 5.139962512709103e-05, + "loss": 0.5553075408935547, + "mean_token_accuracy": 0.8319492520708026, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6361582756042481, + "epoch": 3.213642213642214, + "grad_norm": 1.0183900594711304, + "learning_rate": 5.050073038306314e-05, + "loss": 0.5097637939453125, + "mean_token_accuracy": 0.8410317724943162, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6419171839952469, + "epoch": 3.3423423423423424, + "grad_norm": 1.3186038732528687, + "learning_rate": 4.955940015028897e-05, + "loss": 0.5204424285888671, + "mean_token_accuracy": 0.8418322193622589, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6460894507169723, + "epoch": 3.471042471042471, + "grad_norm": 1.401528000831604, + "learning_rate": 4.857752905996332e-05, + "loss": 0.5238306427001953, + "mean_token_accuracy": 0.8391665583848953, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6538929349184036, + "epoch": 3.5997425997425996, + "grad_norm": 1.3003088235855103, + "learning_rate": 4.7557093340546974e-05, + "loss": 0.5259903335571289, + "mean_token_accuracy": 0.8376457345485687, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6378953090310097, + "epoch": 3.7284427284427286, + "grad_norm": 1.0874862670898438, + "learning_rate": 4.650014684017843e-05, + "loss": 0.5255831527709961, + "mean_token_accuracy": 0.8387808799743652, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6492014318704605, + "epoch": 3.857142857142857, + "grad_norm": 1.2991198301315308, + "learning_rate": 4.54088168928592e-05, + "loss": 0.5239232635498047, + "mean_token_accuracy": 0.8401105499267578, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.6277376067638397, + "epoch": 3.985842985842986, + "grad_norm": 1.1950165033340454, + "learning_rate": 4.428530003673287e-05, + "loss": 0.5076271057128906, + "mean_token_accuracy": 0.8449398750066757, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5833789521885901, + "eval_loss": 0.685325026512146, + "eval_mean_token_accuracy": 0.8025367401309849, + "eval_num_tokens": 2108584.0, + "eval_runtime": 77.1217, + "eval_samples_per_second": 20.033, + "eval_steps_per_second": 2.516, + "step": 1556 + }, + { + "entropy": 0.5782366072890734, + "epoch": 4.113256113256114, + "grad_norm": 1.2209588289260864, + "learning_rate": 4.313185759307585e-05, + "loss": 0.4348250198364258, + "mean_token_accuracy": 0.8640800467645279, + "num_tokens": 2165782.0, + "step": 1600 + }, + { + "entropy": 0.5585444357991218, + "epoch": 4.241956241956242, + "grad_norm": 1.197241187095642, + "learning_rate": 4.1950811114898e-05, + "loss": 0.4139577865600586, + "mean_token_accuracy": 0.8694033759832382, + "num_tokens": 2233206.0, + "step": 1650 + }, + { + "entropy": 0.5706399378180503, + "epoch": 4.370656370656371, + "grad_norm": 1.2947982549667358, + "learning_rate": 4.0744537714313805e-05, + "loss": 0.4305301284790039, + "mean_token_accuracy": 0.8646729928255081, + "num_tokens": 2301399.0, + "step": 1700 + }, + { + "entropy": 0.5524284860491753, + "epoch": 4.499356499356499, + "grad_norm": 1.1909213066101074, + "learning_rate": 3.951546527808885e-05, + "loss": 0.4160680389404297, + "mean_token_accuracy": 0.8686420959234238, + "num_tokens": 2370831.0, + "step": 1750 + }, + { + "entropy": 0.5584349581599235, + "epoch": 4.628056628056628, + "grad_norm": 1.1888494491577148, + "learning_rate": 3.826606758099129e-05, + "loss": 0.42005420684814454, + "mean_token_accuracy": 0.8662647187709809, + "num_tokens": 2439168.0, + "step": 1800 + }, + { + "entropy": 0.5665255922079087, + "epoch": 4.756756756756757, + "grad_norm": 1.175541639328003, + "learning_rate": 3.6998859306783757e-05, + "loss": 0.4301765823364258, + "mean_token_accuracy": 0.8630576282739639, + "num_tokens": 2508036.0, + "step": 1850 + }, + { + "entropy": 0.5487046900391579, + "epoch": 4.885456885456885, + "grad_norm": 1.2421096563339233, + "learning_rate": 3.571639098687706e-05, + "loss": 0.4180584335327148, + "mean_token_accuracy": 0.8661109232902526, + "num_tokens": 2577025.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5167025355334135, + "eval_loss": 0.7642861008644104, + "eval_mean_token_accuracy": 0.7974877059459686, + "eval_num_tokens": 2635730.0, + "eval_runtime": 76.8035, + "eval_samples_per_second": 20.116, + "eval_steps_per_second": 2.526, + "step": 1945 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.441482824545344e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f298f9d53609598cf56ba3b881c57739258a0df9 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.06250292706368335, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..22082943b47b75607e062f46806e6e91555103ad --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2334/trainer_state.json @@ -0,0 +1,560 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2334, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.936105194091797, + "epoch": 0.1287001287001287, + "grad_norm": 2.5512146949768066, + "learning_rate": 7.410977855063067e-06, + "loss": 2.765989074707031, + "mean_token_accuracy": 0.5075639402866363, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.4886422049999237, + "epoch": 0.2574002574002574, + "grad_norm": 2.2254061698913574, + "learning_rate": 1.4973200156147831e-05, + "loss": 1.336165313720703, + "mean_token_accuracy": 0.6740516322851181, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 1.1471627122163772, + "epoch": 0.3861003861003861, + "grad_norm": 2.2073140144348145, + "learning_rate": 2.2535422457232594e-05, + "loss": 1.0278042602539061, + "mean_token_accuracy": 0.7280837368965148, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 1.0486015713214873, + "epoch": 0.5148005148005148, + "grad_norm": 1.650594711303711, + "learning_rate": 3.0097644758317354e-05, + "loss": 0.933094253540039, + "mean_token_accuracy": 0.7478678917884827, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9942852139472962, + "epoch": 0.6435006435006435, + "grad_norm": 1.7949399948120117, + "learning_rate": 3.765986705940212e-05, + "loss": 0.8747442626953125, + "mean_token_accuracy": 0.7603094267845154, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.9491432571411133, + "epoch": 0.7722007722007722, + "grad_norm": 1.33268404006958, + "learning_rate": 4.522208936048689e-05, + "loss": 0.8404794311523438, + "mean_token_accuracy": 0.7671167278289794, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9246599394083023, + "epoch": 0.9009009009009009, + "grad_norm": 1.3137660026550293, + "learning_rate": 5.278431166157165e-05, + "loss": 0.8163955688476563, + "mean_token_accuracy": 0.7712420308589936, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7913847528782088, + "eval_loss": 0.7582769393920898, + "eval_mean_token_accuracy": 0.7787086770092089, + "eval_num_tokens": 527146.0, + "eval_runtime": 80.7434, + "eval_samples_per_second": 19.135, + "eval_steps_per_second": 2.403, + "step": 389 + }, + { + "entropy": 0.9051812700550965, + "epoch": 1.0283140283140284, + "grad_norm": 1.3198363780975342, + "learning_rate": 5.883290514810127e-05, + "loss": 0.7887770843505859, + "mean_token_accuracy": 0.7787209345836832, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8626887363195419, + "epoch": 1.157014157014157, + "grad_norm": 1.1772489547729492, + "learning_rate": 5.8791462758850675e-05, + "loss": 0.7505465698242187, + "mean_token_accuracy": 0.7887020426988601, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8474252647161484, + "epoch": 1.2857142857142856, + "grad_norm": 1.0174403190612793, + "learning_rate": 5.8690897984332274e-05, + "loss": 0.7397904968261719, + "mean_token_accuracy": 0.7890202105045319, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.843030309677124, + "epoch": 1.4144144144144144, + "grad_norm": 1.1678341627120972, + "learning_rate": 5.853141323295946e-05, + "loss": 0.7369876098632813, + "mean_token_accuracy": 0.790421934723854, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8115463078022003, + "epoch": 1.5431145431145432, + "grad_norm": 0.8854549527168274, + "learning_rate": 5.831332950237421e-05, + "loss": 0.7081725311279297, + "mean_token_accuracy": 0.7952603316307068, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8311941117048264, + "epoch": 1.6718146718146718, + "grad_norm": 1.171217441558838, + "learning_rate": 5.803708573336975e-05, + "loss": 0.7232355499267578, + "mean_token_accuracy": 0.7947697073221207, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8120440804958343, + "epoch": 1.8005148005148004, + "grad_norm": 1.0333378314971924, + "learning_rate": 5.770323792642697e-05, + "loss": 0.6972620391845703, + "mean_token_accuracy": 0.7987716430425644, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8045441079139709, + "epoch": 1.9292149292149292, + "grad_norm": 1.01642906665802, + "learning_rate": 5.7312458022642996e-05, + "loss": 0.6930023193359375, + "mean_token_accuracy": 0.8009403163194656, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6984818912658495, + "eval_loss": 0.6775499582290649, + "eval_mean_token_accuracy": 0.8020080097557343, + "eval_num_tokens": 1054292.0, + "eval_runtime": 78.3908, + "eval_samples_per_second": 19.709, + "eval_steps_per_second": 2.475, + "step": 778 + }, + { + "entropy": 0.7779398566544659, + "epoch": 2.056628056628057, + "grad_norm": 1.4157040119171143, + "learning_rate": 5.686553255130396e-05, + "loss": 0.6584475708007812, + "mean_token_accuracy": 0.8087991054611977, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.745064823627472, + "epoch": 2.1853281853281854, + "grad_norm": 0.9885228872299194, + "learning_rate": 5.636336104682433e-05, + "loss": 0.618268051147461, + "mean_token_accuracy": 0.8165298491716385, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7321632200479508, + "epoch": 2.314028314028314, + "grad_norm": 1.0271121263504028, + "learning_rate": 5.580695423823882e-05, + "loss": 0.6195047760009765, + "mean_token_accuracy": 0.8162743961811065, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7240813142061233, + "epoch": 2.4427284427284426, + "grad_norm": 0.8744781613349915, + "learning_rate": 5.519743201489109e-05, + "loss": 0.6093913650512696, + "mean_token_accuracy": 0.8179467821121216, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.7273010468482971, + "epoch": 2.571428571428571, + "grad_norm": 1.0161406993865967, + "learning_rate": 5.453602117241361e-05, + "loss": 0.6118500518798828, + "mean_token_accuracy": 0.8183792388439178, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.7150553166866302, + "epoch": 2.7001287001287, + "grad_norm": 0.9795015454292297, + "learning_rate": 5.382405294353544e-05, + "loss": 0.6005525970458985, + "mean_token_accuracy": 0.8210368871688842, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7134475702047348, + "epoch": 2.828828828828829, + "grad_norm": 0.9153344035148621, + "learning_rate": 5.306296031868776e-05, + "loss": 0.6033024978637695, + "mean_token_accuracy": 0.8216253930330276, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7142637860774994, + "epoch": 2.9575289575289574, + "grad_norm": 0.8638553619384766, + "learning_rate": 5.2254275161799905e-05, + "loss": 0.6056064605712891, + "mean_token_accuracy": 0.8204184120893478, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6126322589584232, + "eval_loss": 0.676252007484436, + "eval_mean_token_accuracy": 0.8002185584958067, + "eval_num_tokens": 1581438.0, + "eval_runtime": 78.2302, + "eval_samples_per_second": 19.749, + "eval_steps_per_second": 2.48, + "step": 1167 + }, + { + "entropy": 0.6793721679485205, + "epoch": 3.0849420849420848, + "grad_norm": 1.2363271713256836, + "learning_rate": 5.139962512709103e-05, + "loss": 0.5553075408935547, + "mean_token_accuracy": 0.8319492520708026, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6361582756042481, + "epoch": 3.213642213642214, + "grad_norm": 1.0183900594711304, + "learning_rate": 5.050073038306314e-05, + "loss": 0.5097637939453125, + "mean_token_accuracy": 0.8410317724943162, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6419171839952469, + "epoch": 3.3423423423423424, + "grad_norm": 1.3186038732528687, + "learning_rate": 4.955940015028897e-05, + "loss": 0.5204424285888671, + "mean_token_accuracy": 0.8418322193622589, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6460894507169723, + "epoch": 3.471042471042471, + "grad_norm": 1.401528000831604, + "learning_rate": 4.857752905996332e-05, + "loss": 0.5238306427001953, + "mean_token_accuracy": 0.8391665583848953, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6538929349184036, + "epoch": 3.5997425997425996, + "grad_norm": 1.3003088235855103, + "learning_rate": 4.7557093340546974e-05, + "loss": 0.5259903335571289, + "mean_token_accuracy": 0.8376457345485687, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6378953090310097, + "epoch": 3.7284427284427286, + "grad_norm": 1.0874862670898438, + "learning_rate": 4.650014684017843e-05, + "loss": 0.5255831527709961, + "mean_token_accuracy": 0.8387808799743652, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6492014318704605, + "epoch": 3.857142857142857, + "grad_norm": 1.2991198301315308, + "learning_rate": 4.54088168928592e-05, + "loss": 0.5239232635498047, + "mean_token_accuracy": 0.8401105499267578, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.6277376067638397, + "epoch": 3.985842985842986, + "grad_norm": 1.1950165033340454, + "learning_rate": 4.428530003673287e-05, + "loss": 0.5076271057128906, + "mean_token_accuracy": 0.8449398750066757, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5833789521885901, + "eval_loss": 0.685325026512146, + "eval_mean_token_accuracy": 0.8025367401309849, + "eval_num_tokens": 2108584.0, + "eval_runtime": 77.1217, + "eval_samples_per_second": 20.033, + "eval_steps_per_second": 2.516, + "step": 1556 + }, + { + "entropy": 0.5782366072890734, + "epoch": 4.113256113256114, + "grad_norm": 1.2209588289260864, + "learning_rate": 4.313185759307585e-05, + "loss": 0.4348250198364258, + "mean_token_accuracy": 0.8640800467645279, + "num_tokens": 2165782.0, + "step": 1600 + }, + { + "entropy": 0.5585444357991218, + "epoch": 4.241956241956242, + "grad_norm": 1.197241187095642, + "learning_rate": 4.1950811114898e-05, + "loss": 0.4139577865600586, + "mean_token_accuracy": 0.8694033759832382, + "num_tokens": 2233206.0, + "step": 1650 + }, + { + "entropy": 0.5706399378180503, + "epoch": 4.370656370656371, + "grad_norm": 1.2947982549667358, + "learning_rate": 4.0744537714313805e-05, + "loss": 0.4305301284790039, + "mean_token_accuracy": 0.8646729928255081, + "num_tokens": 2301399.0, + "step": 1700 + }, + { + "entropy": 0.5524284860491753, + "epoch": 4.499356499356499, + "grad_norm": 1.1909213066101074, + "learning_rate": 3.951546527808885e-05, + "loss": 0.4160680389404297, + "mean_token_accuracy": 0.8686420959234238, + "num_tokens": 2370831.0, + "step": 1750 + }, + { + "entropy": 0.5584349581599235, + "epoch": 4.628056628056628, + "grad_norm": 1.1888494491577148, + "learning_rate": 3.826606758099129e-05, + "loss": 0.42005420684814454, + "mean_token_accuracy": 0.8662647187709809, + "num_tokens": 2439168.0, + "step": 1800 + }, + { + "entropy": 0.5665255922079087, + "epoch": 4.756756756756757, + "grad_norm": 1.175541639328003, + "learning_rate": 3.6998859306783757e-05, + "loss": 0.4301765823364258, + "mean_token_accuracy": 0.8630576282739639, + "num_tokens": 2508036.0, + "step": 1850 + }, + { + "entropy": 0.5487046900391579, + "epoch": 4.885456885456885, + "grad_norm": 1.2421096563339233, + "learning_rate": 3.571639098687706e-05, + "loss": 0.4180584335327148, + "mean_token_accuracy": 0.8661109232902526, + "num_tokens": 2577025.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5167025355334135, + "eval_loss": 0.7642861008644104, + "eval_mean_token_accuracy": 0.7974877059459686, + "eval_num_tokens": 2635730.0, + "eval_runtime": 76.8035, + "eval_samples_per_second": 20.116, + "eval_steps_per_second": 2.526, + "step": 1945 + }, + { + "entropy": 0.5617710394088669, + "epoch": 5.012870012870013, + "grad_norm": 1.0199713706970215, + "learning_rate": 3.442124386683266e-05, + "loss": 0.4222822952270508, + "mean_token_accuracy": 0.8667840885393547, + "num_tokens": 2642803.0, + "step": 1950 + }, + { + "entropy": 0.47934893488883973, + "epoch": 5.141570141570142, + "grad_norm": 1.3371273279190063, + "learning_rate": 3.311602471104637e-05, + "loss": 0.3174937629699707, + "mean_token_accuracy": 0.8981259340047836, + "num_tokens": 2708703.0, + "step": 2000 + }, + { + "entropy": 0.4700386866927147, + "epoch": 5.27027027027027, + "grad_norm": 1.547396183013916, + "learning_rate": 3.180336055606976e-05, + "loss": 0.3171871185302734, + "mean_token_accuracy": 0.8981568259000778, + "num_tokens": 2777956.0, + "step": 2050 + }, + { + "entropy": 0.46871873289346694, + "epoch": 5.398970398970399, + "grad_norm": 1.5816134214401245, + "learning_rate": 3.048589342312954e-05, + "loss": 0.32095291137695314, + "mean_token_accuracy": 0.895929081439972, + "num_tokens": 2847032.0, + "step": 2100 + }, + { + "entropy": 0.46639679729938505, + "epoch": 5.527670527670527, + "grad_norm": 1.399958610534668, + "learning_rate": 2.916627500048691e-05, + "loss": 0.319814453125, + "mean_token_accuracy": 0.8977204102277756, + "num_tokens": 2915746.0, + "step": 2150 + }, + { + "entropy": 0.4666120809316635, + "epoch": 5.656370656370656, + "grad_norm": 1.3686078786849976, + "learning_rate": 2.7847161306340097e-05, + "loss": 0.32353179931640624, + "mean_token_accuracy": 0.8963912934064865, + "num_tokens": 2984346.0, + "step": 2200 + }, + { + "entropy": 0.47636645913124087, + "epoch": 5.785070785070785, + "grad_norm": 1.236251950263977, + "learning_rate": 2.6531207343011766e-05, + "loss": 0.3269305038452148, + "mean_token_accuracy": 0.8954908692836762, + "num_tokens": 3051863.0, + "step": 2250 + }, + { + "entropy": 0.4705436763167381, + "epoch": 5.913770913770914, + "grad_norm": 1.6251490116119385, + "learning_rate": 2.5221061753181217e-05, + "loss": 0.3228004837036133, + "mean_token_accuracy": 0.8956130802631378, + "num_tokens": 3119763.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4492740225546139, + "eval_loss": 0.8636208176612854, + "eval_mean_token_accuracy": 0.7954603777718299, + "eval_num_tokens": 3162876.0, + "eval_runtime": 76.2798, + "eval_samples_per_second": 20.254, + "eval_steps_per_second": 2.543, + "step": 2334 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.333565510679296e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f298f9d53609598cf56ba3b881c57739258a0df9 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.06250292706368335, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2ed3deb6068e1b7469869e681d44613e4246d1d8 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2723/trainer_state.json @@ -0,0 +1,651 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2723, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.936105194091797, + "epoch": 0.1287001287001287, + "grad_norm": 2.5512146949768066, + "learning_rate": 7.410977855063067e-06, + "loss": 2.765989074707031, + "mean_token_accuracy": 0.5075639402866363, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.4886422049999237, + "epoch": 0.2574002574002574, + "grad_norm": 2.2254061698913574, + "learning_rate": 1.4973200156147831e-05, + "loss": 1.336165313720703, + "mean_token_accuracy": 0.6740516322851181, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 1.1471627122163772, + "epoch": 0.3861003861003861, + "grad_norm": 2.2073140144348145, + "learning_rate": 2.2535422457232594e-05, + "loss": 1.0278042602539061, + "mean_token_accuracy": 0.7280837368965148, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 1.0486015713214873, + "epoch": 0.5148005148005148, + "grad_norm": 1.650594711303711, + "learning_rate": 3.0097644758317354e-05, + "loss": 0.933094253540039, + "mean_token_accuracy": 0.7478678917884827, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9942852139472962, + "epoch": 0.6435006435006435, + "grad_norm": 1.7949399948120117, + "learning_rate": 3.765986705940212e-05, + "loss": 0.8747442626953125, + "mean_token_accuracy": 0.7603094267845154, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.9491432571411133, + "epoch": 0.7722007722007722, + "grad_norm": 1.33268404006958, + "learning_rate": 4.522208936048689e-05, + "loss": 0.8404794311523438, + "mean_token_accuracy": 0.7671167278289794, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9246599394083023, + "epoch": 0.9009009009009009, + "grad_norm": 1.3137660026550293, + "learning_rate": 5.278431166157165e-05, + "loss": 0.8163955688476563, + "mean_token_accuracy": 0.7712420308589936, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7913847528782088, + "eval_loss": 0.7582769393920898, + "eval_mean_token_accuracy": 0.7787086770092089, + "eval_num_tokens": 527146.0, + "eval_runtime": 80.7434, + "eval_samples_per_second": 19.135, + "eval_steps_per_second": 2.403, + "step": 389 + }, + { + "entropy": 0.9051812700550965, + "epoch": 1.0283140283140284, + "grad_norm": 1.3198363780975342, + "learning_rate": 5.883290514810127e-05, + "loss": 0.7887770843505859, + "mean_token_accuracy": 0.7787209345836832, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8626887363195419, + "epoch": 1.157014157014157, + "grad_norm": 1.1772489547729492, + "learning_rate": 5.8791462758850675e-05, + "loss": 0.7505465698242187, + "mean_token_accuracy": 0.7887020426988601, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8474252647161484, + "epoch": 1.2857142857142856, + "grad_norm": 1.0174403190612793, + "learning_rate": 5.8690897984332274e-05, + "loss": 0.7397904968261719, + "mean_token_accuracy": 0.7890202105045319, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.843030309677124, + "epoch": 1.4144144144144144, + "grad_norm": 1.1678341627120972, + "learning_rate": 5.853141323295946e-05, + "loss": 0.7369876098632813, + "mean_token_accuracy": 0.790421934723854, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8115463078022003, + "epoch": 1.5431145431145432, + "grad_norm": 0.8854549527168274, + "learning_rate": 5.831332950237421e-05, + "loss": 0.7081725311279297, + "mean_token_accuracy": 0.7952603316307068, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8311941117048264, + "epoch": 1.6718146718146718, + "grad_norm": 1.171217441558838, + "learning_rate": 5.803708573336975e-05, + "loss": 0.7232355499267578, + "mean_token_accuracy": 0.7947697073221207, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8120440804958343, + "epoch": 1.8005148005148004, + "grad_norm": 1.0333378314971924, + "learning_rate": 5.770323792642697e-05, + "loss": 0.6972620391845703, + "mean_token_accuracy": 0.7987716430425644, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8045441079139709, + "epoch": 1.9292149292149292, + "grad_norm": 1.01642906665802, + "learning_rate": 5.7312458022642996e-05, + "loss": 0.6930023193359375, + "mean_token_accuracy": 0.8009403163194656, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6984818912658495, + "eval_loss": 0.6775499582290649, + "eval_mean_token_accuracy": 0.8020080097557343, + "eval_num_tokens": 1054292.0, + "eval_runtime": 78.3908, + "eval_samples_per_second": 19.709, + "eval_steps_per_second": 2.475, + "step": 778 + }, + { + "entropy": 0.7779398566544659, + "epoch": 2.056628056628057, + "grad_norm": 1.4157040119171143, + "learning_rate": 5.686553255130396e-05, + "loss": 0.6584475708007812, + "mean_token_accuracy": 0.8087991054611977, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.745064823627472, + "epoch": 2.1853281853281854, + "grad_norm": 0.9885228872299194, + "learning_rate": 5.636336104682433e-05, + "loss": 0.618268051147461, + "mean_token_accuracy": 0.8165298491716385, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7321632200479508, + "epoch": 2.314028314028314, + "grad_norm": 1.0271121263504028, + "learning_rate": 5.580695423823882e-05, + "loss": 0.6195047760009765, + "mean_token_accuracy": 0.8162743961811065, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7240813142061233, + "epoch": 2.4427284427284426, + "grad_norm": 0.8744781613349915, + "learning_rate": 5.519743201489109e-05, + "loss": 0.6093913650512696, + "mean_token_accuracy": 0.8179467821121216, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.7273010468482971, + "epoch": 2.571428571428571, + "grad_norm": 1.0161406993865967, + "learning_rate": 5.453602117241361e-05, + "loss": 0.6118500518798828, + "mean_token_accuracy": 0.8183792388439178, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.7150553166866302, + "epoch": 2.7001287001287, + "grad_norm": 0.9795015454292297, + "learning_rate": 5.382405294353544e-05, + "loss": 0.6005525970458985, + "mean_token_accuracy": 0.8210368871688842, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7134475702047348, + "epoch": 2.828828828828829, + "grad_norm": 0.9153344035148621, + "learning_rate": 5.306296031868776e-05, + "loss": 0.6033024978637695, + "mean_token_accuracy": 0.8216253930330276, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7142637860774994, + "epoch": 2.9575289575289574, + "grad_norm": 0.8638553619384766, + "learning_rate": 5.2254275161799905e-05, + "loss": 0.6056064605712891, + "mean_token_accuracy": 0.8204184120893478, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6126322589584232, + "eval_loss": 0.676252007484436, + "eval_mean_token_accuracy": 0.8002185584958067, + "eval_num_tokens": 1581438.0, + "eval_runtime": 78.2302, + "eval_samples_per_second": 19.749, + "eval_steps_per_second": 2.48, + "step": 1167 + }, + { + "entropy": 0.6793721679485205, + "epoch": 3.0849420849420848, + "grad_norm": 1.2363271713256836, + "learning_rate": 5.139962512709103e-05, + "loss": 0.5553075408935547, + "mean_token_accuracy": 0.8319492520708026, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6361582756042481, + "epoch": 3.213642213642214, + "grad_norm": 1.0183900594711304, + "learning_rate": 5.050073038306314e-05, + "loss": 0.5097637939453125, + "mean_token_accuracy": 0.8410317724943162, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6419171839952469, + "epoch": 3.3423423423423424, + "grad_norm": 1.3186038732528687, + "learning_rate": 4.955940015028897e-05, + "loss": 0.5204424285888671, + "mean_token_accuracy": 0.8418322193622589, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6460894507169723, + "epoch": 3.471042471042471, + "grad_norm": 1.401528000831604, + "learning_rate": 4.857752905996332e-05, + "loss": 0.5238306427001953, + "mean_token_accuracy": 0.8391665583848953, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6538929349184036, + "epoch": 3.5997425997425996, + "grad_norm": 1.3003088235855103, + "learning_rate": 4.7557093340546974e-05, + "loss": 0.5259903335571289, + "mean_token_accuracy": 0.8376457345485687, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6378953090310097, + "epoch": 3.7284427284427286, + "grad_norm": 1.0874862670898438, + "learning_rate": 4.650014684017843e-05, + "loss": 0.5255831527709961, + "mean_token_accuracy": 0.8387808799743652, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6492014318704605, + "epoch": 3.857142857142857, + "grad_norm": 1.2991198301315308, + "learning_rate": 4.54088168928592e-05, + "loss": 0.5239232635498047, + "mean_token_accuracy": 0.8401105499267578, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.6277376067638397, + "epoch": 3.985842985842986, + "grad_norm": 1.1950165033340454, + "learning_rate": 4.428530003673287e-05, + "loss": 0.5076271057128906, + "mean_token_accuracy": 0.8449398750066757, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5833789521885901, + "eval_loss": 0.685325026512146, + "eval_mean_token_accuracy": 0.8025367401309849, + "eval_num_tokens": 2108584.0, + "eval_runtime": 77.1217, + "eval_samples_per_second": 20.033, + "eval_steps_per_second": 2.516, + "step": 1556 + }, + { + "entropy": 0.5782366072890734, + "epoch": 4.113256113256114, + "grad_norm": 1.2209588289260864, + "learning_rate": 4.313185759307585e-05, + "loss": 0.4348250198364258, + "mean_token_accuracy": 0.8640800467645279, + "num_tokens": 2165782.0, + "step": 1600 + }, + { + "entropy": 0.5585444357991218, + "epoch": 4.241956241956242, + "grad_norm": 1.197241187095642, + "learning_rate": 4.1950811114898e-05, + "loss": 0.4139577865600586, + "mean_token_accuracy": 0.8694033759832382, + "num_tokens": 2233206.0, + "step": 1650 + }, + { + "entropy": 0.5706399378180503, + "epoch": 4.370656370656371, + "grad_norm": 1.2947982549667358, + "learning_rate": 4.0744537714313805e-05, + "loss": 0.4305301284790039, + "mean_token_accuracy": 0.8646729928255081, + "num_tokens": 2301399.0, + "step": 1700 + }, + { + "entropy": 0.5524284860491753, + "epoch": 4.499356499356499, + "grad_norm": 1.1909213066101074, + "learning_rate": 3.951546527808885e-05, + "loss": 0.4160680389404297, + "mean_token_accuracy": 0.8686420959234238, + "num_tokens": 2370831.0, + "step": 1750 + }, + { + "entropy": 0.5584349581599235, + "epoch": 4.628056628056628, + "grad_norm": 1.1888494491577148, + "learning_rate": 3.826606758099129e-05, + "loss": 0.42005420684814454, + "mean_token_accuracy": 0.8662647187709809, + "num_tokens": 2439168.0, + "step": 1800 + }, + { + "entropy": 0.5665255922079087, + "epoch": 4.756756756756757, + "grad_norm": 1.175541639328003, + "learning_rate": 3.6998859306783757e-05, + "loss": 0.4301765823364258, + "mean_token_accuracy": 0.8630576282739639, + "num_tokens": 2508036.0, + "step": 1850 + }, + { + "entropy": 0.5487046900391579, + "epoch": 4.885456885456885, + "grad_norm": 1.2421096563339233, + "learning_rate": 3.571639098687706e-05, + "loss": 0.4180584335327148, + "mean_token_accuracy": 0.8661109232902526, + "num_tokens": 2577025.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5167025355334135, + "eval_loss": 0.7642861008644104, + "eval_mean_token_accuracy": 0.7974877059459686, + "eval_num_tokens": 2635730.0, + "eval_runtime": 76.8035, + "eval_samples_per_second": 20.116, + "eval_steps_per_second": 2.526, + "step": 1945 + }, + { + "entropy": 0.5617710394088669, + "epoch": 5.012870012870013, + "grad_norm": 1.0199713706970215, + "learning_rate": 3.442124386683266e-05, + "loss": 0.4222822952270508, + "mean_token_accuracy": 0.8667840885393547, + "num_tokens": 2642803.0, + "step": 1950 + }, + { + "entropy": 0.47934893488883973, + "epoch": 5.141570141570142, + "grad_norm": 1.3371273279190063, + "learning_rate": 3.311602471104637e-05, + "loss": 0.3174937629699707, + "mean_token_accuracy": 0.8981259340047836, + "num_tokens": 2708703.0, + "step": 2000 + }, + { + "entropy": 0.4700386866927147, + "epoch": 5.27027027027027, + "grad_norm": 1.547396183013916, + "learning_rate": 3.180336055606976e-05, + "loss": 0.3171871185302734, + "mean_token_accuracy": 0.8981568259000778, + "num_tokens": 2777956.0, + "step": 2050 + }, + { + "entropy": 0.46871873289346694, + "epoch": 5.398970398970399, + "grad_norm": 1.5816134214401245, + "learning_rate": 3.048589342312954e-05, + "loss": 0.32095291137695314, + "mean_token_accuracy": 0.895929081439972, + "num_tokens": 2847032.0, + "step": 2100 + }, + { + "entropy": 0.46639679729938505, + "epoch": 5.527670527670527, + "grad_norm": 1.399958610534668, + "learning_rate": 2.916627500048691e-05, + "loss": 0.319814453125, + "mean_token_accuracy": 0.8977204102277756, + "num_tokens": 2915746.0, + "step": 2150 + }, + { + "entropy": 0.4666120809316635, + "epoch": 5.656370656370656, + "grad_norm": 1.3686078786849976, + "learning_rate": 2.7847161306340097e-05, + "loss": 0.32353179931640624, + "mean_token_accuracy": 0.8963912934064865, + "num_tokens": 2984346.0, + "step": 2200 + }, + { + "entropy": 0.47636645913124087, + "epoch": 5.785070785070785, + "grad_norm": 1.236251950263977, + "learning_rate": 2.6531207343011766e-05, + "loss": 0.3269305038452148, + "mean_token_accuracy": 0.8954908692836762, + "num_tokens": 3051863.0, + "step": 2250 + }, + { + "entropy": 0.4705436763167381, + "epoch": 5.913770913770914, + "grad_norm": 1.6251490116119385, + "learning_rate": 2.5221061753181217e-05, + "loss": 0.3228004837036133, + "mean_token_accuracy": 0.8956130802631378, + "num_tokens": 3119763.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4492740225546139, + "eval_loss": 0.8636208176612854, + "eval_mean_token_accuracy": 0.7954603777718299, + "eval_num_tokens": 3162876.0, + "eval_runtime": 76.2798, + "eval_samples_per_second": 20.254, + "eval_steps_per_second": 2.543, + "step": 2334 + }, + { + "entropy": 0.44460387181754063, + "epoch": 6.041184041184041, + "grad_norm": 1.5105853080749512, + "learning_rate": 2.39193614889167e-05, + "loss": 0.29241228103637695, + "mean_token_accuracy": 0.9056352021718266, + "num_tokens": 3185495.0, + "step": 2350 + }, + { + "entropy": 0.3844873097538948, + "epoch": 6.1698841698841695, + "grad_norm": 1.5209016799926758, + "learning_rate": 2.2628726504237633e-05, + "loss": 0.22799573898315428, + "mean_token_accuracy": 0.927648566365242, + "num_tokens": 3252659.0, + "step": 2400 + }, + { + "entropy": 0.392407289147377, + "epoch": 6.298584298584299, + "grad_norm": 1.6858901977539062, + "learning_rate": 2.1351754481889077e-05, + "loss": 0.2331450080871582, + "mean_token_accuracy": 0.9254494923353195, + "num_tokens": 3320155.0, + "step": 2450 + }, + { + "entropy": 0.38693774431943895, + "epoch": 6.427284427284428, + "grad_norm": 1.6617335081100464, + "learning_rate": 2.0091015604941934e-05, + "loss": 0.23076419830322265, + "mean_token_accuracy": 0.925791158080101, + "num_tokens": 3384147.0, + "step": 2500 + }, + { + "entropy": 0.38052063405513764, + "epoch": 6.555984555984556, + "grad_norm": 1.4996752738952637, + "learning_rate": 1.8849047383742318e-05, + "loss": 0.2318257713317871, + "mean_token_accuracy": 0.9256613802909851, + "num_tokens": 3453826.0, + "step": 2550 + }, + { + "entropy": 0.385300210416317, + "epoch": 6.684684684684685, + "grad_norm": 1.5002042055130005, + "learning_rate": 1.7628349548621728e-05, + "loss": 0.2307085418701172, + "mean_token_accuracy": 0.92681689620018, + "num_tokens": 3521099.0, + "step": 2600 + }, + { + "entropy": 0.3840153792500496, + "epoch": 6.813384813384813, + "grad_norm": 1.5271340608596802, + "learning_rate": 1.643137901864781e-05, + "loss": 0.23205507278442383, + "mean_token_accuracy": 0.9271845388412475, + "num_tokens": 3588515.0, + "step": 2650 + }, + { + "entropy": 0.3726115870475769, + "epoch": 6.942084942084942, + "grad_norm": 1.6175596714019775, + "learning_rate": 1.5260544956541976e-05, + "loss": 0.22824867248535155, + "mean_token_accuracy": 0.9268244814872741, + "num_tokens": 3658319.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.39724316409568194, + "eval_loss": 1.0122966766357422, + "eval_mean_token_accuracy": 0.7908440573313802, + "eval_num_tokens": 3690022.0, + "eval_runtime": 76.5271, + "eval_samples_per_second": 20.189, + "eval_steps_per_second": 2.535, + "step": 2723 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.221380441428288e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f298f9d53609598cf56ba3b881c57739258a0df9 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.06250292706368335, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3ed5fc1ecfdda883cd62a8126fa2f37ef3eb600e --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3112/trainer_state.json @@ -0,0 +1,742 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3112, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.936105194091797, + "epoch": 0.1287001287001287, + "grad_norm": 2.5512146949768066, + "learning_rate": 7.410977855063067e-06, + "loss": 2.765989074707031, + "mean_token_accuracy": 0.5075639402866363, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.4886422049999237, + "epoch": 0.2574002574002574, + "grad_norm": 2.2254061698913574, + "learning_rate": 1.4973200156147831e-05, + "loss": 1.336165313720703, + "mean_token_accuracy": 0.6740516322851181, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 1.1471627122163772, + "epoch": 0.3861003861003861, + "grad_norm": 2.2073140144348145, + "learning_rate": 2.2535422457232594e-05, + "loss": 1.0278042602539061, + "mean_token_accuracy": 0.7280837368965148, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 1.0486015713214873, + "epoch": 0.5148005148005148, + "grad_norm": 1.650594711303711, + "learning_rate": 3.0097644758317354e-05, + "loss": 0.933094253540039, + "mean_token_accuracy": 0.7478678917884827, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9942852139472962, + "epoch": 0.6435006435006435, + "grad_norm": 1.7949399948120117, + "learning_rate": 3.765986705940212e-05, + "loss": 0.8747442626953125, + "mean_token_accuracy": 0.7603094267845154, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.9491432571411133, + "epoch": 0.7722007722007722, + "grad_norm": 1.33268404006958, + "learning_rate": 4.522208936048689e-05, + "loss": 0.8404794311523438, + "mean_token_accuracy": 0.7671167278289794, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9246599394083023, + "epoch": 0.9009009009009009, + "grad_norm": 1.3137660026550293, + "learning_rate": 5.278431166157165e-05, + "loss": 0.8163955688476563, + "mean_token_accuracy": 0.7712420308589936, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7913847528782088, + "eval_loss": 0.7582769393920898, + "eval_mean_token_accuracy": 0.7787086770092089, + "eval_num_tokens": 527146.0, + "eval_runtime": 80.7434, + "eval_samples_per_second": 19.135, + "eval_steps_per_second": 2.403, + "step": 389 + }, + { + "entropy": 0.9051812700550965, + "epoch": 1.0283140283140284, + "grad_norm": 1.3198363780975342, + "learning_rate": 5.883290514810127e-05, + "loss": 0.7887770843505859, + "mean_token_accuracy": 0.7787209345836832, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8626887363195419, + "epoch": 1.157014157014157, + "grad_norm": 1.1772489547729492, + "learning_rate": 5.8791462758850675e-05, + "loss": 0.7505465698242187, + "mean_token_accuracy": 0.7887020426988601, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8474252647161484, + "epoch": 1.2857142857142856, + "grad_norm": 1.0174403190612793, + "learning_rate": 5.8690897984332274e-05, + "loss": 0.7397904968261719, + "mean_token_accuracy": 0.7890202105045319, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.843030309677124, + "epoch": 1.4144144144144144, + "grad_norm": 1.1678341627120972, + "learning_rate": 5.853141323295946e-05, + "loss": 0.7369876098632813, + "mean_token_accuracy": 0.790421934723854, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8115463078022003, + "epoch": 1.5431145431145432, + "grad_norm": 0.8854549527168274, + "learning_rate": 5.831332950237421e-05, + "loss": 0.7081725311279297, + "mean_token_accuracy": 0.7952603316307068, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8311941117048264, + "epoch": 1.6718146718146718, + "grad_norm": 1.171217441558838, + "learning_rate": 5.803708573336975e-05, + "loss": 0.7232355499267578, + "mean_token_accuracy": 0.7947697073221207, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8120440804958343, + "epoch": 1.8005148005148004, + "grad_norm": 1.0333378314971924, + "learning_rate": 5.770323792642697e-05, + "loss": 0.6972620391845703, + "mean_token_accuracy": 0.7987716430425644, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8045441079139709, + "epoch": 1.9292149292149292, + "grad_norm": 1.01642906665802, + "learning_rate": 5.7312458022642996e-05, + "loss": 0.6930023193359375, + "mean_token_accuracy": 0.8009403163194656, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6984818912658495, + "eval_loss": 0.6775499582290649, + "eval_mean_token_accuracy": 0.8020080097557343, + "eval_num_tokens": 1054292.0, + "eval_runtime": 78.3908, + "eval_samples_per_second": 19.709, + "eval_steps_per_second": 2.475, + "step": 778 + }, + { + "entropy": 0.7779398566544659, + "epoch": 2.056628056628057, + "grad_norm": 1.4157040119171143, + "learning_rate": 5.686553255130396e-05, + "loss": 0.6584475708007812, + "mean_token_accuracy": 0.8087991054611977, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.745064823627472, + "epoch": 2.1853281853281854, + "grad_norm": 0.9885228872299194, + "learning_rate": 5.636336104682433e-05, + "loss": 0.618268051147461, + "mean_token_accuracy": 0.8165298491716385, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7321632200479508, + "epoch": 2.314028314028314, + "grad_norm": 1.0271121263504028, + "learning_rate": 5.580695423823882e-05, + "loss": 0.6195047760009765, + "mean_token_accuracy": 0.8162743961811065, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7240813142061233, + "epoch": 2.4427284427284426, + "grad_norm": 0.8744781613349915, + "learning_rate": 5.519743201489109e-05, + "loss": 0.6093913650512696, + "mean_token_accuracy": 0.8179467821121216, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.7273010468482971, + "epoch": 2.571428571428571, + "grad_norm": 1.0161406993865967, + "learning_rate": 5.453602117241361e-05, + "loss": 0.6118500518798828, + "mean_token_accuracy": 0.8183792388439178, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.7150553166866302, + "epoch": 2.7001287001287, + "grad_norm": 0.9795015454292297, + "learning_rate": 5.382405294353544e-05, + "loss": 0.6005525970458985, + "mean_token_accuracy": 0.8210368871688842, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7134475702047348, + "epoch": 2.828828828828829, + "grad_norm": 0.9153344035148621, + "learning_rate": 5.306296031868776e-05, + "loss": 0.6033024978637695, + "mean_token_accuracy": 0.8216253930330276, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7142637860774994, + "epoch": 2.9575289575289574, + "grad_norm": 0.8638553619384766, + "learning_rate": 5.2254275161799905e-05, + "loss": 0.6056064605712891, + "mean_token_accuracy": 0.8204184120893478, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6126322589584232, + "eval_loss": 0.676252007484436, + "eval_mean_token_accuracy": 0.8002185584958067, + "eval_num_tokens": 1581438.0, + "eval_runtime": 78.2302, + "eval_samples_per_second": 19.749, + "eval_steps_per_second": 2.48, + "step": 1167 + }, + { + "entropy": 0.6793721679485205, + "epoch": 3.0849420849420848, + "grad_norm": 1.2363271713256836, + "learning_rate": 5.139962512709103e-05, + "loss": 0.5553075408935547, + "mean_token_accuracy": 0.8319492520708026, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6361582756042481, + "epoch": 3.213642213642214, + "grad_norm": 1.0183900594711304, + "learning_rate": 5.050073038306314e-05, + "loss": 0.5097637939453125, + "mean_token_accuracy": 0.8410317724943162, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6419171839952469, + "epoch": 3.3423423423423424, + "grad_norm": 1.3186038732528687, + "learning_rate": 4.955940015028897e-05, + "loss": 0.5204424285888671, + "mean_token_accuracy": 0.8418322193622589, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6460894507169723, + "epoch": 3.471042471042471, + "grad_norm": 1.401528000831604, + "learning_rate": 4.857752905996332e-05, + "loss": 0.5238306427001953, + "mean_token_accuracy": 0.8391665583848953, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6538929349184036, + "epoch": 3.5997425997425996, + "grad_norm": 1.3003088235855103, + "learning_rate": 4.7557093340546974e-05, + "loss": 0.5259903335571289, + "mean_token_accuracy": 0.8376457345485687, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6378953090310097, + "epoch": 3.7284427284427286, + "grad_norm": 1.0874862670898438, + "learning_rate": 4.650014684017843e-05, + "loss": 0.5255831527709961, + "mean_token_accuracy": 0.8387808799743652, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6492014318704605, + "epoch": 3.857142857142857, + "grad_norm": 1.2991198301315308, + "learning_rate": 4.54088168928592e-05, + "loss": 0.5239232635498047, + "mean_token_accuracy": 0.8401105499267578, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.6277376067638397, + "epoch": 3.985842985842986, + "grad_norm": 1.1950165033340454, + "learning_rate": 4.428530003673287e-05, + "loss": 0.5076271057128906, + "mean_token_accuracy": 0.8449398750066757, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5833789521885901, + "eval_loss": 0.685325026512146, + "eval_mean_token_accuracy": 0.8025367401309849, + "eval_num_tokens": 2108584.0, + "eval_runtime": 77.1217, + "eval_samples_per_second": 20.033, + "eval_steps_per_second": 2.516, + "step": 1556 + }, + { + "entropy": 0.5782366072890734, + "epoch": 4.113256113256114, + "grad_norm": 1.2209588289260864, + "learning_rate": 4.313185759307585e-05, + "loss": 0.4348250198364258, + "mean_token_accuracy": 0.8640800467645279, + "num_tokens": 2165782.0, + "step": 1600 + }, + { + "entropy": 0.5585444357991218, + "epoch": 4.241956241956242, + "grad_norm": 1.197241187095642, + "learning_rate": 4.1950811114898e-05, + "loss": 0.4139577865600586, + "mean_token_accuracy": 0.8694033759832382, + "num_tokens": 2233206.0, + "step": 1650 + }, + { + "entropy": 0.5706399378180503, + "epoch": 4.370656370656371, + "grad_norm": 1.2947982549667358, + "learning_rate": 4.0744537714313805e-05, + "loss": 0.4305301284790039, + "mean_token_accuracy": 0.8646729928255081, + "num_tokens": 2301399.0, + "step": 1700 + }, + { + "entropy": 0.5524284860491753, + "epoch": 4.499356499356499, + "grad_norm": 1.1909213066101074, + "learning_rate": 3.951546527808885e-05, + "loss": 0.4160680389404297, + "mean_token_accuracy": 0.8686420959234238, + "num_tokens": 2370831.0, + "step": 1750 + }, + { + "entropy": 0.5584349581599235, + "epoch": 4.628056628056628, + "grad_norm": 1.1888494491577148, + "learning_rate": 3.826606758099129e-05, + "loss": 0.42005420684814454, + "mean_token_accuracy": 0.8662647187709809, + "num_tokens": 2439168.0, + "step": 1800 + }, + { + "entropy": 0.5665255922079087, + "epoch": 4.756756756756757, + "grad_norm": 1.175541639328003, + "learning_rate": 3.6998859306783757e-05, + "loss": 0.4301765823364258, + "mean_token_accuracy": 0.8630576282739639, + "num_tokens": 2508036.0, + "step": 1850 + }, + { + "entropy": 0.5487046900391579, + "epoch": 4.885456885456885, + "grad_norm": 1.2421096563339233, + "learning_rate": 3.571639098687706e-05, + "loss": 0.4180584335327148, + "mean_token_accuracy": 0.8661109232902526, + "num_tokens": 2577025.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5167025355334135, + "eval_loss": 0.7642861008644104, + "eval_mean_token_accuracy": 0.7974877059459686, + "eval_num_tokens": 2635730.0, + "eval_runtime": 76.8035, + "eval_samples_per_second": 20.116, + "eval_steps_per_second": 2.526, + "step": 1945 + }, + { + "entropy": 0.5617710394088669, + "epoch": 5.012870012870013, + "grad_norm": 1.0199713706970215, + "learning_rate": 3.442124386683266e-05, + "loss": 0.4222822952270508, + "mean_token_accuracy": 0.8667840885393547, + "num_tokens": 2642803.0, + "step": 1950 + }, + { + "entropy": 0.47934893488883973, + "epoch": 5.141570141570142, + "grad_norm": 1.3371273279190063, + "learning_rate": 3.311602471104637e-05, + "loss": 0.3174937629699707, + "mean_token_accuracy": 0.8981259340047836, + "num_tokens": 2708703.0, + "step": 2000 + }, + { + "entropy": 0.4700386866927147, + "epoch": 5.27027027027027, + "grad_norm": 1.547396183013916, + "learning_rate": 3.180336055606976e-05, + "loss": 0.3171871185302734, + "mean_token_accuracy": 0.8981568259000778, + "num_tokens": 2777956.0, + "step": 2050 + }, + { + "entropy": 0.46871873289346694, + "epoch": 5.398970398970399, + "grad_norm": 1.5816134214401245, + "learning_rate": 3.048589342312954e-05, + "loss": 0.32095291137695314, + "mean_token_accuracy": 0.895929081439972, + "num_tokens": 2847032.0, + "step": 2100 + }, + { + "entropy": 0.46639679729938505, + "epoch": 5.527670527670527, + "grad_norm": 1.399958610534668, + "learning_rate": 2.916627500048691e-05, + "loss": 0.319814453125, + "mean_token_accuracy": 0.8977204102277756, + "num_tokens": 2915746.0, + "step": 2150 + }, + { + "entropy": 0.4666120809316635, + "epoch": 5.656370656370656, + "grad_norm": 1.3686078786849976, + "learning_rate": 2.7847161306340097e-05, + "loss": 0.32353179931640624, + "mean_token_accuracy": 0.8963912934064865, + "num_tokens": 2984346.0, + "step": 2200 + }, + { + "entropy": 0.47636645913124087, + "epoch": 5.785070785070785, + "grad_norm": 1.236251950263977, + "learning_rate": 2.6531207343011766e-05, + "loss": 0.3269305038452148, + "mean_token_accuracy": 0.8954908692836762, + "num_tokens": 3051863.0, + "step": 2250 + }, + { + "entropy": 0.4705436763167381, + "epoch": 5.913770913770914, + "grad_norm": 1.6251490116119385, + "learning_rate": 2.5221061753181217e-05, + "loss": 0.3228004837036133, + "mean_token_accuracy": 0.8956130802631378, + "num_tokens": 3119763.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4492740225546139, + "eval_loss": 0.8636208176612854, + "eval_mean_token_accuracy": 0.7954603777718299, + "eval_num_tokens": 3162876.0, + "eval_runtime": 76.2798, + "eval_samples_per_second": 20.254, + "eval_steps_per_second": 2.543, + "step": 2334 + }, + { + "entropy": 0.44460387181754063, + "epoch": 6.041184041184041, + "grad_norm": 1.5105853080749512, + "learning_rate": 2.39193614889167e-05, + "loss": 0.29241228103637695, + "mean_token_accuracy": 0.9056352021718266, + "num_tokens": 3185495.0, + "step": 2350 + }, + { + "entropy": 0.3844873097538948, + "epoch": 6.1698841698841695, + "grad_norm": 1.5209016799926758, + "learning_rate": 2.2628726504237633e-05, + "loss": 0.22799573898315428, + "mean_token_accuracy": 0.927648566365242, + "num_tokens": 3252659.0, + "step": 2400 + }, + { + "entropy": 0.392407289147377, + "epoch": 6.298584298584299, + "grad_norm": 1.6858901977539062, + "learning_rate": 2.1351754481889077e-05, + "loss": 0.2331450080871582, + "mean_token_accuracy": 0.9254494923353195, + "num_tokens": 3320155.0, + "step": 2450 + }, + { + "entropy": 0.38693774431943895, + "epoch": 6.427284427284428, + "grad_norm": 1.6617335081100464, + "learning_rate": 2.0091015604941934e-05, + "loss": 0.23076419830322265, + "mean_token_accuracy": 0.925791158080101, + "num_tokens": 3384147.0, + "step": 2500 + }, + { + "entropy": 0.38052063405513764, + "epoch": 6.555984555984556, + "grad_norm": 1.4996752738952637, + "learning_rate": 1.8849047383742318e-05, + "loss": 0.2318257713317871, + "mean_token_accuracy": 0.9256613802909851, + "num_tokens": 3453826.0, + "step": 2550 + }, + { + "entropy": 0.385300210416317, + "epoch": 6.684684684684685, + "grad_norm": 1.5002042055130005, + "learning_rate": 1.7628349548621728e-05, + "loss": 0.2307085418701172, + "mean_token_accuracy": 0.92681689620018, + "num_tokens": 3521099.0, + "step": 2600 + }, + { + "entropy": 0.3840153792500496, + "epoch": 6.813384813384813, + "grad_norm": 1.5271340608596802, + "learning_rate": 1.643137901864781e-05, + "loss": 0.23205507278442383, + "mean_token_accuracy": 0.9271845388412475, + "num_tokens": 3588515.0, + "step": 2650 + }, + { + "entropy": 0.3726115870475769, + "epoch": 6.942084942084942, + "grad_norm": 1.6175596714019775, + "learning_rate": 1.5260544956541976e-05, + "loss": 0.22824867248535155, + "mean_token_accuracy": 0.9268244814872741, + "num_tokens": 3658319.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.39724316409568194, + "eval_loss": 1.0122966766357422, + "eval_mean_token_accuracy": 0.7908440573313802, + "eval_num_tokens": 3690022.0, + "eval_runtime": 76.5271, + "eval_samples_per_second": 20.189, + "eval_steps_per_second": 2.535, + "step": 2723 + }, + { + "entropy": 0.3482914756644856, + "epoch": 7.06949806949807, + "grad_norm": 1.8440797328948975, + "learning_rate": 1.41182039197171e-05, + "loss": 0.19614641189575197, + "mean_token_accuracy": 0.9377202518058546, + "num_tokens": 3727811.0, + "step": 2750 + }, + { + "entropy": 0.32119433909654616, + "epoch": 7.198198198198198, + "grad_norm": 1.3623098134994507, + "learning_rate": 1.3006655117194725e-05, + "loss": 0.16811855316162108, + "mean_token_accuracy": 0.9467526686191559, + "num_tokens": 3795920.0, + "step": 2800 + }, + { + "entropy": 0.31999744042754175, + "epoch": 7.326898326898327, + "grad_norm": 1.2613496780395508, + "learning_rate": 1.192813578194847e-05, + "loss": 0.16953100204467775, + "mean_token_accuracy": 0.9468796044588089, + "num_tokens": 3863324.0, + "step": 2850 + }, + { + "entropy": 0.3202561654150486, + "epoch": 7.455598455598455, + "grad_norm": 1.5667129755020142, + "learning_rate": 1.0884816667987492e-05, + "loss": 0.170334529876709, + "mean_token_accuracy": 0.946151785850525, + "num_tokens": 3932191.0, + "step": 2900 + }, + { + "entropy": 0.3102050364017487, + "epoch": 7.584298584298584, + "grad_norm": 1.5574101209640503, + "learning_rate": 9.878797681243435e-06, + "loss": 0.1649233055114746, + "mean_token_accuracy": 0.9489536666870118, + "num_tokens": 4000632.0, + "step": 2950 + }, + { + "entropy": 0.3167197050154209, + "epoch": 7.712998712998713, + "grad_norm": 1.7108056545257568, + "learning_rate": 8.912103653054436e-06, + "loss": 0.17199958801269533, + "mean_token_accuracy": 0.9465802705287933, + "num_tokens": 4067753.0, + "step": 3000 + }, + { + "entropy": 0.3204824218153954, + "epoch": 7.841698841698841, + "grad_norm": 1.2969859838485718, + "learning_rate": 7.986680264752995e-06, + "loss": 0.17367393493652344, + "mean_token_accuracy": 0.9467671322822571, + "num_tokens": 4133681.0, + "step": 3050 + }, + { + "entropy": 0.31609862834215163, + "epoch": 7.97039897039897, + "grad_norm": 1.4735357761383057, + "learning_rate": 7.104390131560449e-06, + "loss": 0.17134517669677735, + "mean_token_accuracy": 0.946572055220604, + "num_tokens": 4201263.0, + "step": 3100 + }, + { + "epoch": 8.0, + "eval_entropy": 0.36047679170505287, + "eval_loss": 1.1659044027328491, + "eval_mean_token_accuracy": 0.7877745981683436, + "eval_num_tokens": 4217168.0, + "eval_runtime": 76.3532, + "eval_samples_per_second": 20.235, + "eval_steps_per_second": 2.541, + "step": 3112 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.107294599809728e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f298f9d53609598cf56ba3b881c57739258a0df9 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.06250292706368335, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..27ce887dbafde7614a76fd573753205ba675bc84 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3501/trainer_state.json @@ -0,0 +1,833 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3501, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.936105194091797, + "epoch": 0.1287001287001287, + "grad_norm": 2.5512146949768066, + "learning_rate": 7.410977855063067e-06, + "loss": 2.765989074707031, + "mean_token_accuracy": 0.5075639402866363, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.4886422049999237, + "epoch": 0.2574002574002574, + "grad_norm": 2.2254061698913574, + "learning_rate": 1.4973200156147831e-05, + "loss": 1.336165313720703, + "mean_token_accuracy": 0.6740516322851181, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 1.1471627122163772, + "epoch": 0.3861003861003861, + "grad_norm": 2.2073140144348145, + "learning_rate": 2.2535422457232594e-05, + "loss": 1.0278042602539061, + "mean_token_accuracy": 0.7280837368965148, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 1.0486015713214873, + "epoch": 0.5148005148005148, + "grad_norm": 1.650594711303711, + "learning_rate": 3.0097644758317354e-05, + "loss": 0.933094253540039, + "mean_token_accuracy": 0.7478678917884827, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9942852139472962, + "epoch": 0.6435006435006435, + "grad_norm": 1.7949399948120117, + "learning_rate": 3.765986705940212e-05, + "loss": 0.8747442626953125, + "mean_token_accuracy": 0.7603094267845154, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.9491432571411133, + "epoch": 0.7722007722007722, + "grad_norm": 1.33268404006958, + "learning_rate": 4.522208936048689e-05, + "loss": 0.8404794311523438, + "mean_token_accuracy": 0.7671167278289794, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9246599394083023, + "epoch": 0.9009009009009009, + "grad_norm": 1.3137660026550293, + "learning_rate": 5.278431166157165e-05, + "loss": 0.8163955688476563, + "mean_token_accuracy": 0.7712420308589936, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7913847528782088, + "eval_loss": 0.7582769393920898, + "eval_mean_token_accuracy": 0.7787086770092089, + "eval_num_tokens": 527146.0, + "eval_runtime": 80.7434, + "eval_samples_per_second": 19.135, + "eval_steps_per_second": 2.403, + "step": 389 + }, + { + "entropy": 0.9051812700550965, + "epoch": 1.0283140283140284, + "grad_norm": 1.3198363780975342, + "learning_rate": 5.883290514810127e-05, + "loss": 0.7887770843505859, + "mean_token_accuracy": 0.7787209345836832, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8626887363195419, + "epoch": 1.157014157014157, + "grad_norm": 1.1772489547729492, + "learning_rate": 5.8791462758850675e-05, + "loss": 0.7505465698242187, + "mean_token_accuracy": 0.7887020426988601, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8474252647161484, + "epoch": 1.2857142857142856, + "grad_norm": 1.0174403190612793, + "learning_rate": 5.8690897984332274e-05, + "loss": 0.7397904968261719, + "mean_token_accuracy": 0.7890202105045319, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.843030309677124, + "epoch": 1.4144144144144144, + "grad_norm": 1.1678341627120972, + "learning_rate": 5.853141323295946e-05, + "loss": 0.7369876098632813, + "mean_token_accuracy": 0.790421934723854, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8115463078022003, + "epoch": 1.5431145431145432, + "grad_norm": 0.8854549527168274, + "learning_rate": 5.831332950237421e-05, + "loss": 0.7081725311279297, + "mean_token_accuracy": 0.7952603316307068, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8311941117048264, + "epoch": 1.6718146718146718, + "grad_norm": 1.171217441558838, + "learning_rate": 5.803708573336975e-05, + "loss": 0.7232355499267578, + "mean_token_accuracy": 0.7947697073221207, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8120440804958343, + "epoch": 1.8005148005148004, + "grad_norm": 1.0333378314971924, + "learning_rate": 5.770323792642697e-05, + "loss": 0.6972620391845703, + "mean_token_accuracy": 0.7987716430425644, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8045441079139709, + "epoch": 1.9292149292149292, + "grad_norm": 1.01642906665802, + "learning_rate": 5.7312458022642996e-05, + "loss": 0.6930023193359375, + "mean_token_accuracy": 0.8009403163194656, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6984818912658495, + "eval_loss": 0.6775499582290649, + "eval_mean_token_accuracy": 0.8020080097557343, + "eval_num_tokens": 1054292.0, + "eval_runtime": 78.3908, + "eval_samples_per_second": 19.709, + "eval_steps_per_second": 2.475, + "step": 778 + }, + { + "entropy": 0.7779398566544659, + "epoch": 2.056628056628057, + "grad_norm": 1.4157040119171143, + "learning_rate": 5.686553255130396e-05, + "loss": 0.6584475708007812, + "mean_token_accuracy": 0.8087991054611977, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.745064823627472, + "epoch": 2.1853281853281854, + "grad_norm": 0.9885228872299194, + "learning_rate": 5.636336104682433e-05, + "loss": 0.618268051147461, + "mean_token_accuracy": 0.8165298491716385, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7321632200479508, + "epoch": 2.314028314028314, + "grad_norm": 1.0271121263504028, + "learning_rate": 5.580695423823882e-05, + "loss": 0.6195047760009765, + "mean_token_accuracy": 0.8162743961811065, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7240813142061233, + "epoch": 2.4427284427284426, + "grad_norm": 0.8744781613349915, + "learning_rate": 5.519743201489109e-05, + "loss": 0.6093913650512696, + "mean_token_accuracy": 0.8179467821121216, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.7273010468482971, + "epoch": 2.571428571428571, + "grad_norm": 1.0161406993865967, + "learning_rate": 5.453602117241361e-05, + "loss": 0.6118500518798828, + "mean_token_accuracy": 0.8183792388439178, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.7150553166866302, + "epoch": 2.7001287001287, + "grad_norm": 0.9795015454292297, + "learning_rate": 5.382405294353544e-05, + "loss": 0.6005525970458985, + "mean_token_accuracy": 0.8210368871688842, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7134475702047348, + "epoch": 2.828828828828829, + "grad_norm": 0.9153344035148621, + "learning_rate": 5.306296031868776e-05, + "loss": 0.6033024978637695, + "mean_token_accuracy": 0.8216253930330276, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7142637860774994, + "epoch": 2.9575289575289574, + "grad_norm": 0.8638553619384766, + "learning_rate": 5.2254275161799905e-05, + "loss": 0.6056064605712891, + "mean_token_accuracy": 0.8204184120893478, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6126322589584232, + "eval_loss": 0.676252007484436, + "eval_mean_token_accuracy": 0.8002185584958067, + "eval_num_tokens": 1581438.0, + "eval_runtime": 78.2302, + "eval_samples_per_second": 19.749, + "eval_steps_per_second": 2.48, + "step": 1167 + }, + { + "entropy": 0.6793721679485205, + "epoch": 3.0849420849420848, + "grad_norm": 1.2363271713256836, + "learning_rate": 5.139962512709103e-05, + "loss": 0.5553075408935547, + "mean_token_accuracy": 0.8319492520708026, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6361582756042481, + "epoch": 3.213642213642214, + "grad_norm": 1.0183900594711304, + "learning_rate": 5.050073038306314e-05, + "loss": 0.5097637939453125, + "mean_token_accuracy": 0.8410317724943162, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6419171839952469, + "epoch": 3.3423423423423424, + "grad_norm": 1.3186038732528687, + "learning_rate": 4.955940015028897e-05, + "loss": 0.5204424285888671, + "mean_token_accuracy": 0.8418322193622589, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6460894507169723, + "epoch": 3.471042471042471, + "grad_norm": 1.401528000831604, + "learning_rate": 4.857752905996332e-05, + "loss": 0.5238306427001953, + "mean_token_accuracy": 0.8391665583848953, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6538929349184036, + "epoch": 3.5997425997425996, + "grad_norm": 1.3003088235855103, + "learning_rate": 4.7557093340546974e-05, + "loss": 0.5259903335571289, + "mean_token_accuracy": 0.8376457345485687, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6378953090310097, + "epoch": 3.7284427284427286, + "grad_norm": 1.0874862670898438, + "learning_rate": 4.650014684017843e-05, + "loss": 0.5255831527709961, + "mean_token_accuracy": 0.8387808799743652, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6492014318704605, + "epoch": 3.857142857142857, + "grad_norm": 1.2991198301315308, + "learning_rate": 4.54088168928592e-05, + "loss": 0.5239232635498047, + "mean_token_accuracy": 0.8401105499267578, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.6277376067638397, + "epoch": 3.985842985842986, + "grad_norm": 1.1950165033340454, + "learning_rate": 4.428530003673287e-05, + "loss": 0.5076271057128906, + "mean_token_accuracy": 0.8449398750066757, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5833789521885901, + "eval_loss": 0.685325026512146, + "eval_mean_token_accuracy": 0.8025367401309849, + "eval_num_tokens": 2108584.0, + "eval_runtime": 77.1217, + "eval_samples_per_second": 20.033, + "eval_steps_per_second": 2.516, + "step": 1556 + }, + { + "entropy": 0.5782366072890734, + "epoch": 4.113256113256114, + "grad_norm": 1.2209588289260864, + "learning_rate": 4.313185759307585e-05, + "loss": 0.4348250198364258, + "mean_token_accuracy": 0.8640800467645279, + "num_tokens": 2165782.0, + "step": 1600 + }, + { + "entropy": 0.5585444357991218, + "epoch": 4.241956241956242, + "grad_norm": 1.197241187095642, + "learning_rate": 4.1950811114898e-05, + "loss": 0.4139577865600586, + "mean_token_accuracy": 0.8694033759832382, + "num_tokens": 2233206.0, + "step": 1650 + }, + { + "entropy": 0.5706399378180503, + "epoch": 4.370656370656371, + "grad_norm": 1.2947982549667358, + "learning_rate": 4.0744537714313805e-05, + "loss": 0.4305301284790039, + "mean_token_accuracy": 0.8646729928255081, + "num_tokens": 2301399.0, + "step": 1700 + }, + { + "entropy": 0.5524284860491753, + "epoch": 4.499356499356499, + "grad_norm": 1.1909213066101074, + "learning_rate": 3.951546527808885e-05, + "loss": 0.4160680389404297, + "mean_token_accuracy": 0.8686420959234238, + "num_tokens": 2370831.0, + "step": 1750 + }, + { + "entropy": 0.5584349581599235, + "epoch": 4.628056628056628, + "grad_norm": 1.1888494491577148, + "learning_rate": 3.826606758099129e-05, + "loss": 0.42005420684814454, + "mean_token_accuracy": 0.8662647187709809, + "num_tokens": 2439168.0, + "step": 1800 + }, + { + "entropy": 0.5665255922079087, + "epoch": 4.756756756756757, + "grad_norm": 1.175541639328003, + "learning_rate": 3.6998859306783757e-05, + "loss": 0.4301765823364258, + "mean_token_accuracy": 0.8630576282739639, + "num_tokens": 2508036.0, + "step": 1850 + }, + { + "entropy": 0.5487046900391579, + "epoch": 4.885456885456885, + "grad_norm": 1.2421096563339233, + "learning_rate": 3.571639098687706e-05, + "loss": 0.4180584335327148, + "mean_token_accuracy": 0.8661109232902526, + "num_tokens": 2577025.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5167025355334135, + "eval_loss": 0.7642861008644104, + "eval_mean_token_accuracy": 0.7974877059459686, + "eval_num_tokens": 2635730.0, + "eval_runtime": 76.8035, + "eval_samples_per_second": 20.116, + "eval_steps_per_second": 2.526, + "step": 1945 + }, + { + "entropy": 0.5617710394088669, + "epoch": 5.012870012870013, + "grad_norm": 1.0199713706970215, + "learning_rate": 3.442124386683266e-05, + "loss": 0.4222822952270508, + "mean_token_accuracy": 0.8667840885393547, + "num_tokens": 2642803.0, + "step": 1950 + }, + { + "entropy": 0.47934893488883973, + "epoch": 5.141570141570142, + "grad_norm": 1.3371273279190063, + "learning_rate": 3.311602471104637e-05, + "loss": 0.3174937629699707, + "mean_token_accuracy": 0.8981259340047836, + "num_tokens": 2708703.0, + "step": 2000 + }, + { + "entropy": 0.4700386866927147, + "epoch": 5.27027027027027, + "grad_norm": 1.547396183013916, + "learning_rate": 3.180336055606976e-05, + "loss": 0.3171871185302734, + "mean_token_accuracy": 0.8981568259000778, + "num_tokens": 2777956.0, + "step": 2050 + }, + { + "entropy": 0.46871873289346694, + "epoch": 5.398970398970399, + "grad_norm": 1.5816134214401245, + "learning_rate": 3.048589342312954e-05, + "loss": 0.32095291137695314, + "mean_token_accuracy": 0.895929081439972, + "num_tokens": 2847032.0, + "step": 2100 + }, + { + "entropy": 0.46639679729938505, + "epoch": 5.527670527670527, + "grad_norm": 1.399958610534668, + "learning_rate": 2.916627500048691e-05, + "loss": 0.319814453125, + "mean_token_accuracy": 0.8977204102277756, + "num_tokens": 2915746.0, + "step": 2150 + }, + { + "entropy": 0.4666120809316635, + "epoch": 5.656370656370656, + "grad_norm": 1.3686078786849976, + "learning_rate": 2.7847161306340097e-05, + "loss": 0.32353179931640624, + "mean_token_accuracy": 0.8963912934064865, + "num_tokens": 2984346.0, + "step": 2200 + }, + { + "entropy": 0.47636645913124087, + "epoch": 5.785070785070785, + "grad_norm": 1.236251950263977, + "learning_rate": 2.6531207343011766e-05, + "loss": 0.3269305038452148, + "mean_token_accuracy": 0.8954908692836762, + "num_tokens": 3051863.0, + "step": 2250 + }, + { + "entropy": 0.4705436763167381, + "epoch": 5.913770913770914, + "grad_norm": 1.6251490116119385, + "learning_rate": 2.5221061753181217e-05, + "loss": 0.3228004837036133, + "mean_token_accuracy": 0.8956130802631378, + "num_tokens": 3119763.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4492740225546139, + "eval_loss": 0.8636208176612854, + "eval_mean_token_accuracy": 0.7954603777718299, + "eval_num_tokens": 3162876.0, + "eval_runtime": 76.2798, + "eval_samples_per_second": 20.254, + "eval_steps_per_second": 2.543, + "step": 2334 + }, + { + "entropy": 0.44460387181754063, + "epoch": 6.041184041184041, + "grad_norm": 1.5105853080749512, + "learning_rate": 2.39193614889167e-05, + "loss": 0.29241228103637695, + "mean_token_accuracy": 0.9056352021718266, + "num_tokens": 3185495.0, + "step": 2350 + }, + { + "entropy": 0.3844873097538948, + "epoch": 6.1698841698841695, + "grad_norm": 1.5209016799926758, + "learning_rate": 2.2628726504237633e-05, + "loss": 0.22799573898315428, + "mean_token_accuracy": 0.927648566365242, + "num_tokens": 3252659.0, + "step": 2400 + }, + { + "entropy": 0.392407289147377, + "epoch": 6.298584298584299, + "grad_norm": 1.6858901977539062, + "learning_rate": 2.1351754481889077e-05, + "loss": 0.2331450080871582, + "mean_token_accuracy": 0.9254494923353195, + "num_tokens": 3320155.0, + "step": 2450 + }, + { + "entropy": 0.38693774431943895, + "epoch": 6.427284427284428, + "grad_norm": 1.6617335081100464, + "learning_rate": 2.0091015604941934e-05, + "loss": 0.23076419830322265, + "mean_token_accuracy": 0.925791158080101, + "num_tokens": 3384147.0, + "step": 2500 + }, + { + "entropy": 0.38052063405513764, + "epoch": 6.555984555984556, + "grad_norm": 1.4996752738952637, + "learning_rate": 1.8849047383742318e-05, + "loss": 0.2318257713317871, + "mean_token_accuracy": 0.9256613802909851, + "num_tokens": 3453826.0, + "step": 2550 + }, + { + "entropy": 0.385300210416317, + "epoch": 6.684684684684685, + "grad_norm": 1.5002042055130005, + "learning_rate": 1.7628349548621728e-05, + "loss": 0.2307085418701172, + "mean_token_accuracy": 0.92681689620018, + "num_tokens": 3521099.0, + "step": 2600 + }, + { + "entropy": 0.3840153792500496, + "epoch": 6.813384813384813, + "grad_norm": 1.5271340608596802, + "learning_rate": 1.643137901864781e-05, + "loss": 0.23205507278442383, + "mean_token_accuracy": 0.9271845388412475, + "num_tokens": 3588515.0, + "step": 2650 + }, + { + "entropy": 0.3726115870475769, + "epoch": 6.942084942084942, + "grad_norm": 1.6175596714019775, + "learning_rate": 1.5260544956541976e-05, + "loss": 0.22824867248535155, + "mean_token_accuracy": 0.9268244814872741, + "num_tokens": 3658319.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.39724316409568194, + "eval_loss": 1.0122966766357422, + "eval_mean_token_accuracy": 0.7908440573313802, + "eval_num_tokens": 3690022.0, + "eval_runtime": 76.5271, + "eval_samples_per_second": 20.189, + "eval_steps_per_second": 2.535, + "step": 2723 + }, + { + "entropy": 0.3482914756644856, + "epoch": 7.06949806949807, + "grad_norm": 1.8440797328948975, + "learning_rate": 1.41182039197171e-05, + "loss": 0.19614641189575197, + "mean_token_accuracy": 0.9377202518058546, + "num_tokens": 3727811.0, + "step": 2750 + }, + { + "entropy": 0.32119433909654616, + "epoch": 7.198198198198198, + "grad_norm": 1.3623098134994507, + "learning_rate": 1.3006655117194725e-05, + "loss": 0.16811855316162108, + "mean_token_accuracy": 0.9467526686191559, + "num_tokens": 3795920.0, + "step": 2800 + }, + { + "entropy": 0.31999744042754175, + "epoch": 7.326898326898327, + "grad_norm": 1.2613496780395508, + "learning_rate": 1.192813578194847e-05, + "loss": 0.16953100204467775, + "mean_token_accuracy": 0.9468796044588089, + "num_tokens": 3863324.0, + "step": 2850 + }, + { + "entropy": 0.3202561654150486, + "epoch": 7.455598455598455, + "grad_norm": 1.5667129755020142, + "learning_rate": 1.0884816667987492e-05, + "loss": 0.170334529876709, + "mean_token_accuracy": 0.946151785850525, + "num_tokens": 3932191.0, + "step": 2900 + }, + { + "entropy": 0.3102050364017487, + "epoch": 7.584298584298584, + "grad_norm": 1.5574101209640503, + "learning_rate": 9.878797681243435e-06, + "loss": 0.1649233055114746, + "mean_token_accuracy": 0.9489536666870118, + "num_tokens": 4000632.0, + "step": 2950 + }, + { + "entropy": 0.3167197050154209, + "epoch": 7.712998712998713, + "grad_norm": 1.7108056545257568, + "learning_rate": 8.912103653054436e-06, + "loss": 0.17199958801269533, + "mean_token_accuracy": 0.9465802705287933, + "num_tokens": 4067753.0, + "step": 3000 + }, + { + "entropy": 0.3204824218153954, + "epoch": 7.841698841698841, + "grad_norm": 1.2969859838485718, + "learning_rate": 7.986680264752995e-06, + "loss": 0.17367393493652344, + "mean_token_accuracy": 0.9467671322822571, + "num_tokens": 4133681.0, + "step": 3050 + }, + { + "entropy": 0.31609862834215163, + "epoch": 7.97039897039897, + "grad_norm": 1.4735357761383057, + "learning_rate": 7.104390131560449e-06, + "loss": 0.17134517669677735, + "mean_token_accuracy": 0.946572055220604, + "num_tokens": 4201263.0, + "step": 3100 + }, + { + "epoch": 8.0, + "eval_entropy": 0.36047679170505287, + "eval_loss": 1.1659044027328491, + "eval_mean_token_accuracy": 0.7877745981683436, + "eval_num_tokens": 4217168.0, + "eval_runtime": 76.3532, + "eval_samples_per_second": 20.235, + "eval_steps_per_second": 2.541, + "step": 3112 + }, + { + "entropy": 0.28799005530097266, + "epoch": 8.097812097812097, + "grad_norm": 1.2715504169464111, + "learning_rate": 6.267009053669974e-06, + "loss": 0.14414149284362793, + "mean_token_accuracy": 0.9569591073074726, + "num_tokens": 4268387.0, + "step": 3150 + }, + { + "entropy": 0.27990543410182, + "epoch": 8.226512226512227, + "grad_norm": 1.4164396524429321, + "learning_rate": 5.4762224420636936e-06, + "loss": 0.13733112335205078, + "mean_token_accuracy": 0.957932932972908, + "num_tokens": 4335443.0, + "step": 3200 + }, + { + "entropy": 0.2811958509683609, + "epoch": 8.355212355212355, + "grad_norm": 1.0115498304367065, + "learning_rate": 4.733621926257585e-06, + "loss": 0.13586588859558105, + "mean_token_accuracy": 0.9572638642787933, + "num_tokens": 4405738.0, + "step": 3250 + }, + { + "entropy": 0.281443357616663, + "epoch": 8.483912483912484, + "grad_norm": 1.1134238243103027, + "learning_rate": 4.040702150802034e-06, + "loss": 0.14075773239135742, + "mean_token_accuracy": 0.9555223220586777, + "num_tokens": 4471215.0, + "step": 3300 + }, + { + "entropy": 0.2756052704155445, + "epoch": 8.612612612612612, + "grad_norm": 1.2734570503234863, + "learning_rate": 3.39885776698556e-06, + "loss": 0.13990679740905762, + "mean_token_accuracy": 0.9560010427236557, + "num_tokens": 4538643.0, + "step": 3350 + }, + { + "entropy": 0.28228284925222397, + "epoch": 8.741312741312742, + "grad_norm": 1.966119647026062, + "learning_rate": 2.8093806257967536e-06, + "loss": 0.1435575580596924, + "mean_token_accuracy": 0.9555479890108108, + "num_tokens": 4604547.0, + "step": 3400 + }, + { + "entropy": 0.2728269049525261, + "epoch": 8.87001287001287, + "grad_norm": 1.1735725402832031, + "learning_rate": 2.2734571777939993e-06, + "loss": 0.1348598575592041, + "mean_token_accuracy": 0.9582490503787995, + "num_tokens": 4675035.0, + "step": 3450 + }, + { + "entropy": 0.2770812264084816, + "epoch": 8.998712998712998, + "grad_norm": 1.058206558227539, + "learning_rate": 1.7921660851165482e-06, + "loss": 0.1380317783355713, + "mean_token_accuracy": 0.9573065751791, + "num_tokens": 4743741.0, + "step": 3500 + }, + { + "epoch": 9.0, + "eval_entropy": 0.33686197479975594, + "eval_loss": 1.2760401964187622, + "eval_mean_token_accuracy": 0.7863685691479555, + "eval_num_tokens": 4744314.0, + "eval_runtime": 76.2084, + "eval_samples_per_second": 20.273, + "eval_steps_per_second": 2.546, + "step": 3501 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.99789627842816e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f298f9d53609598cf56ba3b881c57739258a0df9 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.06250292706368335, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..dd089398fc3cafe6751c0841bf43867ca87ec00f --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-389/trainer_state.json @@ -0,0 +1,115 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 389, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.936105194091797, + "epoch": 0.1287001287001287, + "grad_norm": 2.5512146949768066, + "learning_rate": 7.410977855063067e-06, + "loss": 2.765989074707031, + "mean_token_accuracy": 0.5075639402866363, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.4886422049999237, + "epoch": 0.2574002574002574, + "grad_norm": 2.2254061698913574, + "learning_rate": 1.4973200156147831e-05, + "loss": 1.336165313720703, + "mean_token_accuracy": 0.6740516322851181, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 1.1471627122163772, + "epoch": 0.3861003861003861, + "grad_norm": 2.2073140144348145, + "learning_rate": 2.2535422457232594e-05, + "loss": 1.0278042602539061, + "mean_token_accuracy": 0.7280837368965148, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 1.0486015713214873, + "epoch": 0.5148005148005148, + "grad_norm": 1.650594711303711, + "learning_rate": 3.0097644758317354e-05, + "loss": 0.933094253540039, + "mean_token_accuracy": 0.7478678917884827, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9942852139472962, + "epoch": 0.6435006435006435, + "grad_norm": 1.7949399948120117, + "learning_rate": 3.765986705940212e-05, + "loss": 0.8747442626953125, + "mean_token_accuracy": 0.7603094267845154, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.9491432571411133, + "epoch": 0.7722007722007722, + "grad_norm": 1.33268404006958, + "learning_rate": 4.522208936048689e-05, + "loss": 0.8404794311523438, + "mean_token_accuracy": 0.7671167278289794, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9246599394083023, + "epoch": 0.9009009009009009, + "grad_norm": 1.3137660026550293, + "learning_rate": 5.278431166157165e-05, + "loss": 0.8163955688476563, + "mean_token_accuracy": 0.7712420308589936, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7913847528782088, + "eval_loss": 0.7582769393920898, + "eval_mean_token_accuracy": 0.7787086770092089, + "eval_num_tokens": 527146.0, + "eval_runtime": 80.7434, + "eval_samples_per_second": 19.135, + "eval_steps_per_second": 2.403, + "step": 389 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8877868880031360.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f298f9d53609598cf56ba3b881c57739258a0df9 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.06250292706368335, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f4f64e3a6d447322f3e91e0284cba0c6e762e291 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3890/trainer_state.json @@ -0,0 +1,914 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 3890, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.936105194091797, + "epoch": 0.1287001287001287, + "grad_norm": 2.5512146949768066, + "learning_rate": 7.410977855063067e-06, + "loss": 2.765989074707031, + "mean_token_accuracy": 0.5075639402866363, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.4886422049999237, + "epoch": 0.2574002574002574, + "grad_norm": 2.2254061698913574, + "learning_rate": 1.4973200156147831e-05, + "loss": 1.336165313720703, + "mean_token_accuracy": 0.6740516322851181, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 1.1471627122163772, + "epoch": 0.3861003861003861, + "grad_norm": 2.2073140144348145, + "learning_rate": 2.2535422457232594e-05, + "loss": 1.0278042602539061, + "mean_token_accuracy": 0.7280837368965148, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 1.0486015713214873, + "epoch": 0.5148005148005148, + "grad_norm": 1.650594711303711, + "learning_rate": 3.0097644758317354e-05, + "loss": 0.933094253540039, + "mean_token_accuracy": 0.7478678917884827, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9942852139472962, + "epoch": 0.6435006435006435, + "grad_norm": 1.7949399948120117, + "learning_rate": 3.765986705940212e-05, + "loss": 0.8747442626953125, + "mean_token_accuracy": 0.7603094267845154, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.9491432571411133, + "epoch": 0.7722007722007722, + "grad_norm": 1.33268404006958, + "learning_rate": 4.522208936048689e-05, + "loss": 0.8404794311523438, + "mean_token_accuracy": 0.7671167278289794, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9246599394083023, + "epoch": 0.9009009009009009, + "grad_norm": 1.3137660026550293, + "learning_rate": 5.278431166157165e-05, + "loss": 0.8163955688476563, + "mean_token_accuracy": 0.7712420308589936, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7913847528782088, + "eval_loss": 0.7582769393920898, + "eval_mean_token_accuracy": 0.7787086770092089, + "eval_num_tokens": 527146.0, + "eval_runtime": 80.7434, + "eval_samples_per_second": 19.135, + "eval_steps_per_second": 2.403, + "step": 389 + }, + { + "entropy": 0.9051812700550965, + "epoch": 1.0283140283140284, + "grad_norm": 1.3198363780975342, + "learning_rate": 5.883290514810127e-05, + "loss": 0.7887770843505859, + "mean_token_accuracy": 0.7787209345836832, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8626887363195419, + "epoch": 1.157014157014157, + "grad_norm": 1.1772489547729492, + "learning_rate": 5.8791462758850675e-05, + "loss": 0.7505465698242187, + "mean_token_accuracy": 0.7887020426988601, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8474252647161484, + "epoch": 1.2857142857142856, + "grad_norm": 1.0174403190612793, + "learning_rate": 5.8690897984332274e-05, + "loss": 0.7397904968261719, + "mean_token_accuracy": 0.7890202105045319, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.843030309677124, + "epoch": 1.4144144144144144, + "grad_norm": 1.1678341627120972, + "learning_rate": 5.853141323295946e-05, + "loss": 0.7369876098632813, + "mean_token_accuracy": 0.790421934723854, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8115463078022003, + "epoch": 1.5431145431145432, + "grad_norm": 0.8854549527168274, + "learning_rate": 5.831332950237421e-05, + "loss": 0.7081725311279297, + "mean_token_accuracy": 0.7952603316307068, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8311941117048264, + "epoch": 1.6718146718146718, + "grad_norm": 1.171217441558838, + "learning_rate": 5.803708573336975e-05, + "loss": 0.7232355499267578, + "mean_token_accuracy": 0.7947697073221207, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8120440804958343, + "epoch": 1.8005148005148004, + "grad_norm": 1.0333378314971924, + "learning_rate": 5.770323792642697e-05, + "loss": 0.6972620391845703, + "mean_token_accuracy": 0.7987716430425644, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8045441079139709, + "epoch": 1.9292149292149292, + "grad_norm": 1.01642906665802, + "learning_rate": 5.7312458022642996e-05, + "loss": 0.6930023193359375, + "mean_token_accuracy": 0.8009403163194656, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6984818912658495, + "eval_loss": 0.6775499582290649, + "eval_mean_token_accuracy": 0.8020080097557343, + "eval_num_tokens": 1054292.0, + "eval_runtime": 78.3908, + "eval_samples_per_second": 19.709, + "eval_steps_per_second": 2.475, + "step": 778 + }, + { + "entropy": 0.7779398566544659, + "epoch": 2.056628056628057, + "grad_norm": 1.4157040119171143, + "learning_rate": 5.686553255130396e-05, + "loss": 0.6584475708007812, + "mean_token_accuracy": 0.8087991054611977, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.745064823627472, + "epoch": 2.1853281853281854, + "grad_norm": 0.9885228872299194, + "learning_rate": 5.636336104682433e-05, + "loss": 0.618268051147461, + "mean_token_accuracy": 0.8165298491716385, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7321632200479508, + "epoch": 2.314028314028314, + "grad_norm": 1.0271121263504028, + "learning_rate": 5.580695423823882e-05, + "loss": 0.6195047760009765, + "mean_token_accuracy": 0.8162743961811065, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7240813142061233, + "epoch": 2.4427284427284426, + "grad_norm": 0.8744781613349915, + "learning_rate": 5.519743201489109e-05, + "loss": 0.6093913650512696, + "mean_token_accuracy": 0.8179467821121216, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.7273010468482971, + "epoch": 2.571428571428571, + "grad_norm": 1.0161406993865967, + "learning_rate": 5.453602117241361e-05, + "loss": 0.6118500518798828, + "mean_token_accuracy": 0.8183792388439178, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.7150553166866302, + "epoch": 2.7001287001287, + "grad_norm": 0.9795015454292297, + "learning_rate": 5.382405294353544e-05, + "loss": 0.6005525970458985, + "mean_token_accuracy": 0.8210368871688842, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7134475702047348, + "epoch": 2.828828828828829, + "grad_norm": 0.9153344035148621, + "learning_rate": 5.306296031868776e-05, + "loss": 0.6033024978637695, + "mean_token_accuracy": 0.8216253930330276, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7142637860774994, + "epoch": 2.9575289575289574, + "grad_norm": 0.8638553619384766, + "learning_rate": 5.2254275161799905e-05, + "loss": 0.6056064605712891, + "mean_token_accuracy": 0.8204184120893478, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6126322589584232, + "eval_loss": 0.676252007484436, + "eval_mean_token_accuracy": 0.8002185584958067, + "eval_num_tokens": 1581438.0, + "eval_runtime": 78.2302, + "eval_samples_per_second": 19.749, + "eval_steps_per_second": 2.48, + "step": 1167 + }, + { + "entropy": 0.6793721679485205, + "epoch": 3.0849420849420848, + "grad_norm": 1.2363271713256836, + "learning_rate": 5.139962512709103e-05, + "loss": 0.5553075408935547, + "mean_token_accuracy": 0.8319492520708026, + "num_tokens": 1626128.0, + "step": 1200 + }, + { + "entropy": 0.6361582756042481, + "epoch": 3.213642213642214, + "grad_norm": 1.0183900594711304, + "learning_rate": 5.050073038306314e-05, + "loss": 0.5097637939453125, + "mean_token_accuracy": 0.8410317724943162, + "num_tokens": 1694451.0, + "step": 1250 + }, + { + "entropy": 0.6419171839952469, + "epoch": 3.3423423423423424, + "grad_norm": 1.3186038732528687, + "learning_rate": 4.955940015028897e-05, + "loss": 0.5204424285888671, + "mean_token_accuracy": 0.8418322193622589, + "num_tokens": 1763832.0, + "step": 1300 + }, + { + "entropy": 0.6460894507169723, + "epoch": 3.471042471042471, + "grad_norm": 1.401528000831604, + "learning_rate": 4.857752905996332e-05, + "loss": 0.5238306427001953, + "mean_token_accuracy": 0.8391665583848953, + "num_tokens": 1830309.0, + "step": 1350 + }, + { + "entropy": 0.6538929349184036, + "epoch": 3.5997425997425996, + "grad_norm": 1.3003088235855103, + "learning_rate": 4.7557093340546974e-05, + "loss": 0.5259903335571289, + "mean_token_accuracy": 0.8376457345485687, + "num_tokens": 1896710.0, + "step": 1400 + }, + { + "entropy": 0.6378953090310097, + "epoch": 3.7284427284427286, + "grad_norm": 1.0874862670898438, + "learning_rate": 4.650014684017843e-05, + "loss": 0.5255831527709961, + "mean_token_accuracy": 0.8387808799743652, + "num_tokens": 1967081.0, + "step": 1450 + }, + { + "entropy": 0.6492014318704605, + "epoch": 3.857142857142857, + "grad_norm": 1.2991198301315308, + "learning_rate": 4.54088168928592e-05, + "loss": 0.5239232635498047, + "mean_token_accuracy": 0.8401105499267578, + "num_tokens": 2033170.0, + "step": 1500 + }, + { + "entropy": 0.6277376067638397, + "epoch": 3.985842985842986, + "grad_norm": 1.1950165033340454, + "learning_rate": 4.428530003673287e-05, + "loss": 0.5076271057128906, + "mean_token_accuracy": 0.8449398750066757, + "num_tokens": 2101077.0, + "step": 1550 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5833789521885901, + "eval_loss": 0.685325026512146, + "eval_mean_token_accuracy": 0.8025367401309849, + "eval_num_tokens": 2108584.0, + "eval_runtime": 77.1217, + "eval_samples_per_second": 20.033, + "eval_steps_per_second": 2.516, + "step": 1556 + }, + { + "entropy": 0.5782366072890734, + "epoch": 4.113256113256114, + "grad_norm": 1.2209588289260864, + "learning_rate": 4.313185759307585e-05, + "loss": 0.4348250198364258, + "mean_token_accuracy": 0.8640800467645279, + "num_tokens": 2165782.0, + "step": 1600 + }, + { + "entropy": 0.5585444357991218, + "epoch": 4.241956241956242, + "grad_norm": 1.197241187095642, + "learning_rate": 4.1950811114898e-05, + "loss": 0.4139577865600586, + "mean_token_accuracy": 0.8694033759832382, + "num_tokens": 2233206.0, + "step": 1650 + }, + { + "entropy": 0.5706399378180503, + "epoch": 4.370656370656371, + "grad_norm": 1.2947982549667358, + "learning_rate": 4.0744537714313805e-05, + "loss": 0.4305301284790039, + "mean_token_accuracy": 0.8646729928255081, + "num_tokens": 2301399.0, + "step": 1700 + }, + { + "entropy": 0.5524284860491753, + "epoch": 4.499356499356499, + "grad_norm": 1.1909213066101074, + "learning_rate": 3.951546527808885e-05, + "loss": 0.4160680389404297, + "mean_token_accuracy": 0.8686420959234238, + "num_tokens": 2370831.0, + "step": 1750 + }, + { + "entropy": 0.5584349581599235, + "epoch": 4.628056628056628, + "grad_norm": 1.1888494491577148, + "learning_rate": 3.826606758099129e-05, + "loss": 0.42005420684814454, + "mean_token_accuracy": 0.8662647187709809, + "num_tokens": 2439168.0, + "step": 1800 + }, + { + "entropy": 0.5665255922079087, + "epoch": 4.756756756756757, + "grad_norm": 1.175541639328003, + "learning_rate": 3.6998859306783757e-05, + "loss": 0.4301765823364258, + "mean_token_accuracy": 0.8630576282739639, + "num_tokens": 2508036.0, + "step": 1850 + }, + { + "entropy": 0.5487046900391579, + "epoch": 4.885456885456885, + "grad_norm": 1.2421096563339233, + "learning_rate": 3.571639098687706e-05, + "loss": 0.4180584335327148, + "mean_token_accuracy": 0.8661109232902526, + "num_tokens": 2577025.0, + "step": 1900 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5167025355334135, + "eval_loss": 0.7642861008644104, + "eval_mean_token_accuracy": 0.7974877059459686, + "eval_num_tokens": 2635730.0, + "eval_runtime": 76.8035, + "eval_samples_per_second": 20.116, + "eval_steps_per_second": 2.526, + "step": 1945 + }, + { + "entropy": 0.5617710394088669, + "epoch": 5.012870012870013, + "grad_norm": 1.0199713706970215, + "learning_rate": 3.442124386683266e-05, + "loss": 0.4222822952270508, + "mean_token_accuracy": 0.8667840885393547, + "num_tokens": 2642803.0, + "step": 1950 + }, + { + "entropy": 0.47934893488883973, + "epoch": 5.141570141570142, + "grad_norm": 1.3371273279190063, + "learning_rate": 3.311602471104637e-05, + "loss": 0.3174937629699707, + "mean_token_accuracy": 0.8981259340047836, + "num_tokens": 2708703.0, + "step": 2000 + }, + { + "entropy": 0.4700386866927147, + "epoch": 5.27027027027027, + "grad_norm": 1.547396183013916, + "learning_rate": 3.180336055606976e-05, + "loss": 0.3171871185302734, + "mean_token_accuracy": 0.8981568259000778, + "num_tokens": 2777956.0, + "step": 2050 + }, + { + "entropy": 0.46871873289346694, + "epoch": 5.398970398970399, + "grad_norm": 1.5816134214401245, + "learning_rate": 3.048589342312954e-05, + "loss": 0.32095291137695314, + "mean_token_accuracy": 0.895929081439972, + "num_tokens": 2847032.0, + "step": 2100 + }, + { + "entropy": 0.46639679729938505, + "epoch": 5.527670527670527, + "grad_norm": 1.399958610534668, + "learning_rate": 2.916627500048691e-05, + "loss": 0.319814453125, + "mean_token_accuracy": 0.8977204102277756, + "num_tokens": 2915746.0, + "step": 2150 + }, + { + "entropy": 0.4666120809316635, + "epoch": 5.656370656370656, + "grad_norm": 1.3686078786849976, + "learning_rate": 2.7847161306340097e-05, + "loss": 0.32353179931640624, + "mean_token_accuracy": 0.8963912934064865, + "num_tokens": 2984346.0, + "step": 2200 + }, + { + "entropy": 0.47636645913124087, + "epoch": 5.785070785070785, + "grad_norm": 1.236251950263977, + "learning_rate": 2.6531207343011766e-05, + "loss": 0.3269305038452148, + "mean_token_accuracy": 0.8954908692836762, + "num_tokens": 3051863.0, + "step": 2250 + }, + { + "entropy": 0.4705436763167381, + "epoch": 5.913770913770914, + "grad_norm": 1.6251490116119385, + "learning_rate": 2.5221061753181217e-05, + "loss": 0.3228004837036133, + "mean_token_accuracy": 0.8956130802631378, + "num_tokens": 3119763.0, + "step": 2300 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4492740225546139, + "eval_loss": 0.8636208176612854, + "eval_mean_token_accuracy": 0.7954603777718299, + "eval_num_tokens": 3162876.0, + "eval_runtime": 76.2798, + "eval_samples_per_second": 20.254, + "eval_steps_per_second": 2.543, + "step": 2334 + }, + { + "entropy": 0.44460387181754063, + "epoch": 6.041184041184041, + "grad_norm": 1.5105853080749512, + "learning_rate": 2.39193614889167e-05, + "loss": 0.29241228103637695, + "mean_token_accuracy": 0.9056352021718266, + "num_tokens": 3185495.0, + "step": 2350 + }, + { + "entropy": 0.3844873097538948, + "epoch": 6.1698841698841695, + "grad_norm": 1.5209016799926758, + "learning_rate": 2.2628726504237633e-05, + "loss": 0.22799573898315428, + "mean_token_accuracy": 0.927648566365242, + "num_tokens": 3252659.0, + "step": 2400 + }, + { + "entropy": 0.392407289147377, + "epoch": 6.298584298584299, + "grad_norm": 1.6858901977539062, + "learning_rate": 2.1351754481889077e-05, + "loss": 0.2331450080871582, + "mean_token_accuracy": 0.9254494923353195, + "num_tokens": 3320155.0, + "step": 2450 + }, + { + "entropy": 0.38693774431943895, + "epoch": 6.427284427284428, + "grad_norm": 1.6617335081100464, + "learning_rate": 2.0091015604941934e-05, + "loss": 0.23076419830322265, + "mean_token_accuracy": 0.925791158080101, + "num_tokens": 3384147.0, + "step": 2500 + }, + { + "entropy": 0.38052063405513764, + "epoch": 6.555984555984556, + "grad_norm": 1.4996752738952637, + "learning_rate": 1.8849047383742318e-05, + "loss": 0.2318257713317871, + "mean_token_accuracy": 0.9256613802909851, + "num_tokens": 3453826.0, + "step": 2550 + }, + { + "entropy": 0.385300210416317, + "epoch": 6.684684684684685, + "grad_norm": 1.5002042055130005, + "learning_rate": 1.7628349548621728e-05, + "loss": 0.2307085418701172, + "mean_token_accuracy": 0.92681689620018, + "num_tokens": 3521099.0, + "step": 2600 + }, + { + "entropy": 0.3840153792500496, + "epoch": 6.813384813384813, + "grad_norm": 1.5271340608596802, + "learning_rate": 1.643137901864781e-05, + "loss": 0.23205507278442383, + "mean_token_accuracy": 0.9271845388412475, + "num_tokens": 3588515.0, + "step": 2650 + }, + { + "entropy": 0.3726115870475769, + "epoch": 6.942084942084942, + "grad_norm": 1.6175596714019775, + "learning_rate": 1.5260544956541976e-05, + "loss": 0.22824867248535155, + "mean_token_accuracy": 0.9268244814872741, + "num_tokens": 3658319.0, + "step": 2700 + }, + { + "epoch": 7.0, + "eval_entropy": 0.39724316409568194, + "eval_loss": 1.0122966766357422, + "eval_mean_token_accuracy": 0.7908440573313802, + "eval_num_tokens": 3690022.0, + "eval_runtime": 76.5271, + "eval_samples_per_second": 20.189, + "eval_steps_per_second": 2.535, + "step": 2723 + }, + { + "entropy": 0.3482914756644856, + "epoch": 7.06949806949807, + "grad_norm": 1.8440797328948975, + "learning_rate": 1.41182039197171e-05, + "loss": 0.19614641189575197, + "mean_token_accuracy": 0.9377202518058546, + "num_tokens": 3727811.0, + "step": 2750 + }, + { + "entropy": 0.32119433909654616, + "epoch": 7.198198198198198, + "grad_norm": 1.3623098134994507, + "learning_rate": 1.3006655117194725e-05, + "loss": 0.16811855316162108, + "mean_token_accuracy": 0.9467526686191559, + "num_tokens": 3795920.0, + "step": 2800 + }, + { + "entropy": 0.31999744042754175, + "epoch": 7.326898326898327, + "grad_norm": 1.2613496780395508, + "learning_rate": 1.192813578194847e-05, + "loss": 0.16953100204467775, + "mean_token_accuracy": 0.9468796044588089, + "num_tokens": 3863324.0, + "step": 2850 + }, + { + "entropy": 0.3202561654150486, + "epoch": 7.455598455598455, + "grad_norm": 1.5667129755020142, + "learning_rate": 1.0884816667987492e-05, + "loss": 0.170334529876709, + "mean_token_accuracy": 0.946151785850525, + "num_tokens": 3932191.0, + "step": 2900 + }, + { + "entropy": 0.3102050364017487, + "epoch": 7.584298584298584, + "grad_norm": 1.5574101209640503, + "learning_rate": 9.878797681243435e-06, + "loss": 0.1649233055114746, + "mean_token_accuracy": 0.9489536666870118, + "num_tokens": 4000632.0, + "step": 2950 + }, + { + "entropy": 0.3167197050154209, + "epoch": 7.712998712998713, + "grad_norm": 1.7108056545257568, + "learning_rate": 8.912103653054436e-06, + "loss": 0.17199958801269533, + "mean_token_accuracy": 0.9465802705287933, + "num_tokens": 4067753.0, + "step": 3000 + }, + { + "entropy": 0.3204824218153954, + "epoch": 7.841698841698841, + "grad_norm": 1.2969859838485718, + "learning_rate": 7.986680264752995e-06, + "loss": 0.17367393493652344, + "mean_token_accuracy": 0.9467671322822571, + "num_tokens": 4133681.0, + "step": 3050 + }, + { + "entropy": 0.31609862834215163, + "epoch": 7.97039897039897, + "grad_norm": 1.4735357761383057, + "learning_rate": 7.104390131560449e-06, + "loss": 0.17134517669677735, + "mean_token_accuracy": 0.946572055220604, + "num_tokens": 4201263.0, + "step": 3100 + }, + { + "epoch": 8.0, + "eval_entropy": 0.36047679170505287, + "eval_loss": 1.1659044027328491, + "eval_mean_token_accuracy": 0.7877745981683436, + "eval_num_tokens": 4217168.0, + "eval_runtime": 76.3532, + "eval_samples_per_second": 20.235, + "eval_steps_per_second": 2.541, + "step": 3112 + }, + { + "entropy": 0.28799005530097266, + "epoch": 8.097812097812097, + "grad_norm": 1.2715504169464111, + "learning_rate": 6.267009053669974e-06, + "loss": 0.14414149284362793, + "mean_token_accuracy": 0.9569591073074726, + "num_tokens": 4268387.0, + "step": 3150 + }, + { + "entropy": 0.27990543410182, + "epoch": 8.226512226512227, + "grad_norm": 1.4164396524429321, + "learning_rate": 5.4762224420636936e-06, + "loss": 0.13733112335205078, + "mean_token_accuracy": 0.957932932972908, + "num_tokens": 4335443.0, + "step": 3200 + }, + { + "entropy": 0.2811958509683609, + "epoch": 8.355212355212355, + "grad_norm": 1.0115498304367065, + "learning_rate": 4.733621926257585e-06, + "loss": 0.13586588859558105, + "mean_token_accuracy": 0.9572638642787933, + "num_tokens": 4405738.0, + "step": 3250 + }, + { + "entropy": 0.281443357616663, + "epoch": 8.483912483912484, + "grad_norm": 1.1134238243103027, + "learning_rate": 4.040702150802034e-06, + "loss": 0.14075773239135742, + "mean_token_accuracy": 0.9555223220586777, + "num_tokens": 4471215.0, + "step": 3300 + }, + { + "entropy": 0.2756052704155445, + "epoch": 8.612612612612612, + "grad_norm": 1.2734570503234863, + "learning_rate": 3.39885776698556e-06, + "loss": 0.13990679740905762, + "mean_token_accuracy": 0.9560010427236557, + "num_tokens": 4538643.0, + "step": 3350 + }, + { + "entropy": 0.28228284925222397, + "epoch": 8.741312741312742, + "grad_norm": 1.966119647026062, + "learning_rate": 2.8093806257967536e-06, + "loss": 0.1435575580596924, + "mean_token_accuracy": 0.9555479890108108, + "num_tokens": 4604547.0, + "step": 3400 + }, + { + "entropy": 0.2728269049525261, + "epoch": 8.87001287001287, + "grad_norm": 1.1735725402832031, + "learning_rate": 2.2734571777939993e-06, + "loss": 0.1348598575592041, + "mean_token_accuracy": 0.9582490503787995, + "num_tokens": 4675035.0, + "step": 3450 + }, + { + "entropy": 0.2770812264084816, + "epoch": 8.998712998712998, + "grad_norm": 1.058206558227539, + "learning_rate": 1.7921660851165482e-06, + "loss": 0.1380317783355713, + "mean_token_accuracy": 0.9573065751791, + "num_tokens": 4743741.0, + "step": 3500 + }, + { + "epoch": 9.0, + "eval_entropy": 0.33686197479975594, + "eval_loss": 1.2760401964187622, + "eval_mean_token_accuracy": 0.7863685691479555, + "eval_num_tokens": 4744314.0, + "eval_runtime": 76.2084, + "eval_samples_per_second": 20.273, + "eval_steps_per_second": 2.546, + "step": 3501 + }, + { + "entropy": 0.2675129978945761, + "epoch": 9.126126126126126, + "grad_norm": 0.9801173210144043, + "learning_rate": 1.366476050443016e-06, + "loss": 0.12434512138366699, + "mean_token_accuracy": 0.9614468704570424, + "num_tokens": 4810493.0, + "step": 3550 + }, + { + "entropy": 0.2641969184577465, + "epoch": 9.254826254826256, + "grad_norm": 0.8309164047241211, + "learning_rate": 9.9724386726727e-07, + "loss": 0.12468255996704102, + "mean_token_accuracy": 0.9620213270187378, + "num_tokens": 4877111.0, + "step": 3600 + }, + { + "entropy": 0.2615968011319637, + "epoch": 9.383526383526384, + "grad_norm": 0.8614134192466736, + "learning_rate": 6.852126954156949e-07, + "loss": 0.12330841064453125, + "mean_token_accuracy": 0.9619427108764649, + "num_tokens": 4945117.0, + "step": 3650 + }, + { + "entropy": 0.26592432290315626, + "epoch": 9.512226512226512, + "grad_norm": 0.8157131671905518, + "learning_rate": 4.3101056527695085e-07, + "loss": 0.1279654598236084, + "mean_token_accuracy": 0.960343233346939, + "num_tokens": 5012749.0, + "step": 3700 + }, + { + "entropy": 0.26622014299035074, + "epoch": 9.64092664092664, + "grad_norm": 1.1939444541931152, + "learning_rate": 2.351491137546123e-07, + "loss": 0.12640375137329102, + "mean_token_accuracy": 0.9612093043327331, + "num_tokens": 5080092.0, + "step": 3750 + }, + { + "entropy": 0.2676539346575737, + "epoch": 9.76962676962677, + "grad_norm": 0.956229567527771, + "learning_rate": 9.802255448699602e-08, + "loss": 0.1269227981567383, + "mean_token_accuracy": 0.9606389313936233, + "num_tokens": 5147114.0, + "step": 3800 + }, + { + "entropy": 0.2567200756072998, + "epoch": 9.898326898326898, + "grad_norm": 1.040459394454956, + "learning_rate": 1.990688440671166e-08, + "loss": 0.12122024536132812, + "mean_token_accuracy": 0.9627848011255264, + "num_tokens": 5217747.0, + "step": 3850 + }, + { + "epoch": 10.0, + "eval_entropy": 0.3312081219916491, + "eval_loss": 1.3217803239822388, + "eval_mean_token_accuracy": 0.7861843738973755, + "eval_num_tokens": 5271460.0, + "eval_runtime": 76.1487, + "eval_samples_per_second": 20.289, + "eval_steps_per_second": 2.548, + "step": 3890 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 8.885472845837376e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f298f9d53609598cf56ba3b881c57739258a0df9 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.06250292706368335, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6e8ef5e07a698e3fc88f349fb522342911202db5 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-778/trainer_state.json @@ -0,0 +1,206 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 778, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.936105194091797, + "epoch": 0.1287001287001287, + "grad_norm": 2.5512146949768066, + "learning_rate": 7.410977855063067e-06, + "loss": 2.765989074707031, + "mean_token_accuracy": 0.5075639402866363, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.4886422049999237, + "epoch": 0.2574002574002574, + "grad_norm": 2.2254061698913574, + "learning_rate": 1.4973200156147831e-05, + "loss": 1.336165313720703, + "mean_token_accuracy": 0.6740516322851181, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 1.1471627122163772, + "epoch": 0.3861003861003861, + "grad_norm": 2.2073140144348145, + "learning_rate": 2.2535422457232594e-05, + "loss": 1.0278042602539061, + "mean_token_accuracy": 0.7280837368965148, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 1.0486015713214873, + "epoch": 0.5148005148005148, + "grad_norm": 1.650594711303711, + "learning_rate": 3.0097644758317354e-05, + "loss": 0.933094253540039, + "mean_token_accuracy": 0.7478678917884827, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9942852139472962, + "epoch": 0.6435006435006435, + "grad_norm": 1.7949399948120117, + "learning_rate": 3.765986705940212e-05, + "loss": 0.8747442626953125, + "mean_token_accuracy": 0.7603094267845154, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.9491432571411133, + "epoch": 0.7722007722007722, + "grad_norm": 1.33268404006958, + "learning_rate": 4.522208936048689e-05, + "loss": 0.8404794311523438, + "mean_token_accuracy": 0.7671167278289794, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9246599394083023, + "epoch": 0.9009009009009009, + "grad_norm": 1.3137660026550293, + "learning_rate": 5.278431166157165e-05, + "loss": 0.8163955688476563, + "mean_token_accuracy": 0.7712420308589936, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7913847528782088, + "eval_loss": 0.7582769393920898, + "eval_mean_token_accuracy": 0.7787086770092089, + "eval_num_tokens": 527146.0, + "eval_runtime": 80.7434, + "eval_samples_per_second": 19.135, + "eval_steps_per_second": 2.403, + "step": 389 + }, + { + "entropy": 0.9051812700550965, + "epoch": 1.0283140283140284, + "grad_norm": 1.3198363780975342, + "learning_rate": 5.883290514810127e-05, + "loss": 0.7887770843505859, + "mean_token_accuracy": 0.7787209345836832, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8626887363195419, + "epoch": 1.157014157014157, + "grad_norm": 1.1772489547729492, + "learning_rate": 5.8791462758850675e-05, + "loss": 0.7505465698242187, + "mean_token_accuracy": 0.7887020426988601, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8474252647161484, + "epoch": 1.2857142857142856, + "grad_norm": 1.0174403190612793, + "learning_rate": 5.8690897984332274e-05, + "loss": 0.7397904968261719, + "mean_token_accuracy": 0.7890202105045319, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.843030309677124, + "epoch": 1.4144144144144144, + "grad_norm": 1.1678341627120972, + "learning_rate": 5.853141323295946e-05, + "loss": 0.7369876098632813, + "mean_token_accuracy": 0.790421934723854, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8115463078022003, + "epoch": 1.5431145431145432, + "grad_norm": 0.8854549527168274, + "learning_rate": 5.831332950237421e-05, + "loss": 0.7081725311279297, + "mean_token_accuracy": 0.7952603316307068, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8311941117048264, + "epoch": 1.6718146718146718, + "grad_norm": 1.171217441558838, + "learning_rate": 5.803708573336975e-05, + "loss": 0.7232355499267578, + "mean_token_accuracy": 0.7947697073221207, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8120440804958343, + "epoch": 1.8005148005148004, + "grad_norm": 1.0333378314971924, + "learning_rate": 5.770323792642697e-05, + "loss": 0.6972620391845703, + "mean_token_accuracy": 0.7987716430425644, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8045441079139709, + "epoch": 1.9292149292149292, + "grad_norm": 1.01642906665802, + "learning_rate": 5.7312458022642996e-05, + "loss": 0.6930023193359375, + "mean_token_accuracy": 0.8009403163194656, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6984818912658495, + "eval_loss": 0.6775499582290649, + "eval_mean_token_accuracy": 0.8020080097557343, + "eval_num_tokens": 1054292.0, + "eval_runtime": 78.3908, + "eval_samples_per_second": 19.709, + "eval_steps_per_second": 2.475, + "step": 778 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.773786050957952e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}